@omnimedia/omnitool 1.1.0-1 → 1.1.0-11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (190) hide show
  1. package/README.md +1 -1
  2. package/package.json +12 -9
  3. package/s/context.ts +0 -7
  4. package/s/demo/demo.bundle.ts +34 -5
  5. package/s/demo/demo.css +5 -0
  6. package/s/demo/routines/filmstrip-test.ts +2 -2
  7. package/s/demo/routines/transcode-test.ts +4 -2
  8. package/s/demo/routines/transcriber-test.ts +34 -0
  9. package/s/demo/routines/transitions-test.ts +43 -0
  10. package/s/demo/routines/waveform-test.ts +3 -2
  11. package/s/driver/driver.ts +19 -11
  12. package/s/driver/fns/host.ts +7 -6
  13. package/s/driver/fns/schematic.ts +45 -22
  14. package/s/driver/fns/work.ts +163 -151
  15. package/s/driver/utils/load-decoder-source.ts +3 -4
  16. package/s/features/speech/transcribe/default-spec.ts +11 -0
  17. package/s/features/speech/transcribe/parts/load-pipe.ts +19 -0
  18. package/s/features/speech/transcribe/parts/prep-audio.ts +23 -0
  19. package/s/features/speech/transcribe/parts/transcribe.ts +70 -0
  20. package/s/features/speech/transcribe/transcriber.ts +46 -0
  21. package/s/features/speech/transcribe/types.ts +82 -0
  22. package/s/features/speech/transcribe/worker.bundle.ts +40 -0
  23. package/s/features/transition/parts/fragment.ts +24 -0
  24. package/s/features/transition/parts/types.ts +94 -0
  25. package/s/features/transition/parts/uniforms.ts +29 -0
  26. package/s/features/transition/parts/vertex.ts +31 -0
  27. package/s/features/transition/transition.ts +60 -0
  28. package/s/index.html.ts +6 -1
  29. package/s/timeline/index.ts +1 -0
  30. package/s/timeline/parts/basics.ts +1 -1
  31. package/s/timeline/parts/compositor/export.ts +77 -0
  32. package/s/timeline/parts/compositor/parts/html-tree.ts +37 -0
  33. package/s/timeline/parts/compositor/parts/schedulers.ts +85 -0
  34. package/s/timeline/parts/compositor/parts/tree-builder.ts +184 -0
  35. package/s/timeline/parts/compositor/parts/webcodecs-tree.ts +30 -0
  36. package/s/timeline/parts/compositor/playback.ts +80 -0
  37. package/s/timeline/parts/compositor/samplers/html.ts +115 -0
  38. package/s/timeline/parts/compositor/samplers/webcodecs.ts +61 -0
  39. package/s/timeline/parts/item.ts +38 -6
  40. package/s/timeline/parts/media.ts +21 -0
  41. package/s/timeline/parts/waveform.ts +3 -4
  42. package/s/timeline/sugar/builders.ts +102 -0
  43. package/s/timeline/sugar/o.ts +117 -27
  44. package/s/timeline/sugar/omni-test.ts +5 -3
  45. package/s/timeline/sugar/omni.ts +26 -11
  46. package/s/timeline/types.ts +29 -0
  47. package/s/timeline/utils/audio-stream.ts +15 -0
  48. package/s/timeline/utils/checksum.ts +2 -1
  49. package/s/timeline/utils/matrix.ts +33 -0
  50. package/s/timeline/utils/video-cursor.ts +40 -0
  51. package/x/context.d.ts +1 -4
  52. package/x/context.js +1 -5
  53. package/x/context.js.map +1 -1
  54. package/x/demo/demo.bundle.js +21 -5
  55. package/x/demo/demo.bundle.js.map +1 -1
  56. package/x/demo/demo.bundle.min.js +606 -36
  57. package/x/demo/demo.bundle.min.js.map +4 -4
  58. package/x/demo/demo.css +5 -0
  59. package/x/demo/routines/filmstrip-test.d.ts +1 -1
  60. package/x/demo/routines/filmstrip-test.js +2 -2
  61. package/x/demo/routines/filmstrip-test.js.map +1 -1
  62. package/x/demo/routines/transcode-test.js +4 -2
  63. package/x/demo/routines/transcode-test.js.map +1 -1
  64. package/x/demo/routines/transcriber-test.d.ts +4 -0
  65. package/x/demo/routines/transcriber-test.js +33 -0
  66. package/x/demo/routines/transcriber-test.js.map +1 -0
  67. package/x/demo/routines/transitions-test.d.ts +5 -0
  68. package/x/demo/routines/transitions-test.js +35 -0
  69. package/x/demo/routines/transitions-test.js.map +1 -0
  70. package/x/demo/routines/waveform-test.d.ts +2 -1
  71. package/x/demo/routines/waveform-test.js +2 -2
  72. package/x/demo/routines/waveform-test.js.map +1 -1
  73. package/x/driver/driver.d.ts +4 -6
  74. package/x/driver/driver.js +17 -10
  75. package/x/driver/driver.js.map +1 -1
  76. package/x/driver/driver.worker.bundle.min.js +2537 -148
  77. package/x/driver/driver.worker.bundle.min.js.map +4 -4
  78. package/x/driver/fns/host.d.ts +9 -2
  79. package/x/driver/fns/host.js +3 -3
  80. package/x/driver/fns/host.js.map +1 -1
  81. package/x/driver/fns/schematic.d.ts +39 -21
  82. package/x/driver/fns/work.d.ts +11 -4
  83. package/x/driver/fns/work.js +111 -102
  84. package/x/driver/fns/work.js.map +1 -1
  85. package/x/driver/utils/load-decoder-source.d.ts +2 -1
  86. package/x/driver/utils/load-decoder-source.js +2 -3
  87. package/x/driver/utils/load-decoder-source.js.map +1 -1
  88. package/x/features/speech/transcribe/default-spec.d.ts +2 -0
  89. package/x/features/speech/transcribe/default-spec.js +8 -0
  90. package/x/features/speech/transcribe/default-spec.js.map +1 -0
  91. package/x/features/speech/transcribe/parts/load-pipe.d.ts +2 -0
  92. package/x/features/speech/transcribe/parts/load-pipe.js +13 -0
  93. package/x/features/speech/transcribe/parts/load-pipe.js.map +1 -0
  94. package/x/features/speech/transcribe/parts/prep-audio.d.ts +5 -0
  95. package/x/features/speech/transcribe/parts/prep-audio.js +21 -0
  96. package/x/features/speech/transcribe/parts/prep-audio.js.map +1 -0
  97. package/x/features/speech/transcribe/parts/transcribe.d.ts +5 -0
  98. package/x/features/speech/transcribe/parts/transcribe.js +56 -0
  99. package/x/features/speech/transcribe/parts/transcribe.js.map +1 -0
  100. package/x/features/speech/transcribe/transcriber.d.ts +5 -0
  101. package/x/features/speech/transcribe/transcriber.js +33 -0
  102. package/x/features/speech/transcribe/transcriber.js.map +1 -0
  103. package/x/features/speech/transcribe/types.d.ts +66 -0
  104. package/x/features/speech/transcribe/types.js +2 -0
  105. package/x/features/speech/transcribe/types.js.map +1 -0
  106. package/x/features/speech/transcribe/worker.bundle.d.ts +1 -0
  107. package/x/features/speech/transcribe/worker.bundle.js +33 -0
  108. package/x/features/speech/transcribe/worker.bundle.js.map +1 -0
  109. package/x/features/speech/transcribe/worker.bundle.min.js +2916 -0
  110. package/x/features/speech/transcribe/worker.bundle.min.js.map +7 -0
  111. package/x/features/transition/parts/fragment.d.ts +1 -0
  112. package/x/features/transition/parts/fragment.js +25 -0
  113. package/x/features/transition/parts/fragment.js.map +1 -0
  114. package/x/features/transition/parts/types.d.ts +23 -0
  115. package/x/features/transition/parts/types.js +2 -0
  116. package/x/features/transition/parts/types.js.map +1 -0
  117. package/x/features/transition/parts/uniforms.d.ts +31 -0
  118. package/x/features/transition/parts/uniforms.js +27 -0
  119. package/x/features/transition/parts/uniforms.js.map +1 -0
  120. package/x/features/transition/parts/vertex.d.ts +1 -0
  121. package/x/features/transition/parts/vertex.js +32 -0
  122. package/x/features/transition/parts/vertex.js.map +1 -0
  123. package/x/features/transition/transition.d.ts +5 -0
  124. package/x/features/transition/transition.js +50 -0
  125. package/x/features/transition/transition.js.map +1 -0
  126. package/x/index.html +13 -3
  127. package/x/index.html.js +6 -1
  128. package/x/index.html.js.map +1 -1
  129. package/x/timeline/index.d.ts +1 -0
  130. package/x/timeline/index.js +1 -0
  131. package/x/timeline/index.js.map +1 -1
  132. package/x/timeline/parts/basics.d.ts +1 -1
  133. package/x/timeline/parts/compositor/export.d.ts +11 -0
  134. package/x/timeline/parts/compositor/export.js +64 -0
  135. package/x/timeline/parts/compositor/export.js.map +1 -0
  136. package/x/timeline/parts/compositor/parts/html-tree.d.ts +3 -0
  137. package/x/timeline/parts/compositor/parts/html-tree.js +40 -0
  138. package/x/timeline/parts/compositor/parts/html-tree.js.map +1 -0
  139. package/x/timeline/parts/compositor/parts/schedulers.d.ts +15 -0
  140. package/x/timeline/parts/compositor/parts/schedulers.js +64 -0
  141. package/x/timeline/parts/compositor/parts/schedulers.js.map +1 -0
  142. package/x/timeline/parts/compositor/parts/tree-builder.d.ts +37 -0
  143. package/x/timeline/parts/compositor/parts/tree-builder.js +147 -0
  144. package/x/timeline/parts/compositor/parts/tree-builder.js.map +1 -0
  145. package/x/timeline/parts/compositor/parts/webcodecs-tree.d.ts +3 -0
  146. package/x/timeline/parts/compositor/parts/webcodecs-tree.js +28 -0
  147. package/x/timeline/parts/compositor/parts/webcodecs-tree.js.map +1 -0
  148. package/x/timeline/parts/compositor/playback.d.ts +21 -0
  149. package/x/timeline/parts/compositor/playback.js +70 -0
  150. package/x/timeline/parts/compositor/playback.js.map +1 -0
  151. package/x/timeline/parts/compositor/samplers/html.d.ts +3 -0
  152. package/x/timeline/parts/compositor/samplers/html.js +106 -0
  153. package/x/timeline/parts/compositor/samplers/html.js.map +1 -0
  154. package/x/timeline/parts/compositor/samplers/webcodecs.d.ts +3 -0
  155. package/x/timeline/parts/compositor/samplers/webcodecs.js +52 -0
  156. package/x/timeline/parts/compositor/samplers/webcodecs.js.map +1 -0
  157. package/x/timeline/parts/item.d.ts +34 -8
  158. package/x/timeline/parts/item.js +6 -3
  159. package/x/timeline/parts/item.js.map +1 -1
  160. package/x/timeline/parts/media.d.ts +3 -0
  161. package/x/timeline/parts/media.js +17 -0
  162. package/x/timeline/parts/media.js.map +1 -1
  163. package/x/timeline/parts/waveform.d.ts +2 -1
  164. package/x/timeline/parts/waveform.js +2 -4
  165. package/x/timeline/parts/waveform.js.map +1 -1
  166. package/x/timeline/sugar/builders.d.ts +1 -0
  167. package/x/timeline/sugar/builders.js +104 -0
  168. package/x/timeline/sugar/builders.js.map +1 -0
  169. package/x/timeline/sugar/o.d.ts +23 -5
  170. package/x/timeline/sugar/o.js +93 -27
  171. package/x/timeline/sugar/o.js.map +1 -1
  172. package/x/timeline/sugar/omni-test.js +4 -2
  173. package/x/timeline/sugar/omni-test.js.map +1 -1
  174. package/x/timeline/sugar/omni.d.ts +8 -2
  175. package/x/timeline/sugar/omni.js +22 -9
  176. package/x/timeline/sugar/omni.js.map +1 -1
  177. package/x/timeline/types.d.ts +24 -0
  178. package/x/timeline/types.js +2 -0
  179. package/x/timeline/types.js.map +1 -0
  180. package/x/timeline/utils/audio-stream.d.ts +6 -0
  181. package/x/timeline/utils/audio-stream.js +17 -0
  182. package/x/timeline/utils/audio-stream.js.map +1 -0
  183. package/x/timeline/utils/checksum.js +2 -1
  184. package/x/timeline/utils/checksum.js.map +1 -1
  185. package/x/timeline/utils/matrix.d.ts +8 -0
  186. package/x/timeline/utils/matrix.js +26 -0
  187. package/x/timeline/utils/matrix.js.map +1 -0
  188. package/x/timeline/utils/video-cursor.d.ts +10 -0
  189. package/x/timeline/utils/video-cursor.js +36 -0
  190. package/x/timeline/utils/video-cursor.js.map +1 -0
@@ -0,0 +1,102 @@
1
+ // import {O} from "./o.js"
2
+ // import {Id} from "../parts/basics.js"
3
+ // import {Item} from "../parts/item.js"
4
+ //
5
+ // export class TimelineItem {
6
+ // public readonly id: Id
7
+ //
8
+ // constructor(public item: Item.Any) {
9
+ // this.id = item.id
10
+ // }
11
+ //
12
+ // toJSON() {
13
+ // return {
14
+ // ...this.item
15
+ // }
16
+ // }
17
+ // }
18
+ //
19
+ // abstract class VisualItem extends TimelineItem {
20
+ // abstract spatial(spatial: Spatial): TimelineItem
21
+ // }
22
+ //
23
+ // export class Stack extends VisualItem {
24
+ // constructor(private o: O, public item: Item.Stack) {
25
+ // super(item)
26
+ // }
27
+ //
28
+ // spatial(spatial: Spatial) {
29
+ // this.item.spatialId = spatial.item.id
30
+ // return this
31
+ // }
32
+ //
33
+ // addChildren(fn: (o: O) => TimelineItem | TimelineItem[]) {
34
+ // const result = fn(this.o)
35
+ // const items = Array.isArray(result) ? result : [result]
36
+ // this.item.childrenIds.push(...items.map(c => c.item.id))
37
+ // return this
38
+ // }
39
+ // }
40
+ //
41
+ // export class Spatial extends TimelineItem {
42
+ // constructor(public item: Item.Spatial) {super(item)}
43
+ // }
44
+ //
45
+ // export class Gap extends TimelineItem {
46
+ // constructor(public item: Item.Gap) {super(item)}
47
+ // }
48
+ //
49
+ // export class Audio extends TimelineItem {
50
+ // constructor(public item: Item.Audio) {super(item)}
51
+ // }
52
+ //
53
+ // export class Video extends VisualItem {
54
+ // constructor(public item: Item.Video) {
55
+ // super(item)
56
+ // }
57
+ //
58
+ // spatial(spatial: Spatial) {
59
+ // this.item.spatialId = spatial.item.id
60
+ // return this
61
+ // }
62
+ // }
63
+ //
64
+ // export class Text extends VisualItem {
65
+ // constructor(public item: Item.Text) {
66
+ // super(item)
67
+ // }
68
+ //
69
+ // color(color: string) {
70
+ // this.item.color = color
71
+ // return this
72
+ // }
73
+ //
74
+ // spatial(spatial: Spatial) {
75
+ // this.item.spatialId = spatial.item.id
76
+ // return this
77
+ // }
78
+ // }
79
+ //
80
+ // export class Sequence extends VisualItem {
81
+ // constructor(private o: O, public item: Item.Sequence) {
82
+ // super(item)
83
+ // }
84
+ //
85
+ // spatial(spatial: Spatial) {
86
+ // this.item.spatialId = spatial.item.id
87
+ // return this
88
+ // }
89
+ //
90
+ // addChildren(fn: (o: O) => TimelineItem | TimelineItem[]) {
91
+ // const result = fn(this.o)
92
+ // const items = Array.isArray(result) ? result : [result]
93
+ // this.item.childrenIds.push(...items.map(c => c.item.id))
94
+ // return this
95
+ // }
96
+ // }
97
+ //
98
+ // export class Transition extends TimelineItem {
99
+ // constructor(public item: Item.Transition) {
100
+ // super(item)
101
+ // }
102
+ // }
@@ -1,51 +1,119 @@
1
1
 
2
- import {MapG} from "@e280/stz"
3
- import {Id} from "../parts/basics.js"
4
2
  import {Media} from "../parts/media.js"
3
+ import {Id, TimelineFile} from "../parts/basics.js"
5
4
  import {Effect, Item, Kind} from "../parts/item.js"
5
+ import {Transform, TransformOptions, Vec2} from "../types.js"
6
6
 
7
7
  export class O {
8
8
  #nextId = 0
9
- #items = new MapG<Id, Item.Any>()
9
+
10
+ constructor(public state: {project: TimelineFile}) {}
11
+
12
+ require<T extends Item.Any>(id: Id): T {
13
+ const item = this.state.project.items.find(item => item.id === id)
14
+ return item as T
15
+ }
10
16
 
11
17
  #getId() {
12
18
  return this.#nextId++
13
19
  }
14
20
 
15
- register(item: Item.Any) {
16
- if (!this.#items.has(item.id))
17
- this.#items.set(item.id, item)
18
- return item.id
21
+ #mutate(fn: (project: TimelineFile) => TimelineFile) {
22
+ this.state.project = fn(this.state.project)
23
+ }
24
+
25
+ spatial = (transform: Transform): Item.Spatial => {
26
+ const item: Item.Spatial = {
27
+ id: this.#getId(),
28
+ kind: Kind.Spatial,
29
+ transform
30
+ }
31
+ return item
32
+ }
33
+
34
+ sequence = (...items: Item.Any[]): Item.Any => {
35
+ const item = {
36
+ id: this.#getId(),
37
+ kind: Kind.Sequence,
38
+ childrenIds: items.map(item => item.id)
39
+ } as Item.Sequence
40
+
41
+ this.#mutate(state => {
42
+ state.items.push(item, ...items)
43
+ return state
44
+ })
45
+
46
+ return item
19
47
  }
20
48
 
21
- get items() {
22
- return [...this.#items.values()]
49
+ stack = (...items: Item.Any[]): Item.Any => {
50
+ const item = {
51
+ kind: Kind.Stack,
52
+ id: this.#getId(),
53
+ childrenIds: items.map(item => item.id)
54
+ } as Item.Stack
55
+
56
+ this.#mutate(state => {
57
+ state.items.push(item, ...items)
58
+ return state
59
+ })
60
+
61
+ return item
23
62
  }
24
63
 
25
- sequence = (...items: Item.Any[]): Item.Sequence => ({
26
- id: this.#getId(),
27
- kind: Kind.Sequence,
28
- children: items.map(item => this.register(item)),
29
- })
64
+ video = (
65
+ media: Media,
66
+ options?: {
67
+ start?: number,
68
+ duration?: number
69
+ }): Item.Video => {
30
70
 
31
- stack = (...items: Item.Any[]): Item.Stack => ({
32
- id: this.#getId(),
33
- kind: Kind.Stack,
34
- children: items.map(item => this.register(item)),
35
- })
71
+ if(!media.hasVideo)
72
+ throw new Error(`Video clip error: media "${media.datafile.filename}" has no video track.`)
36
73
 
37
- clip = (media: Media, start?: number, duration?: number): Item.Clip => ({
38
- id: this.#getId(),
39
- kind: Kind.Clip,
40
- mediaHash: media.datafile.checksum.hash,
41
- start: start ?? 0,
42
- duration: duration ?? media.duration,
43
- })
74
+ const item: Item.Video = {
75
+ kind: Kind.Video,
76
+ id: this.#getId(),
77
+ mediaHash: media.datafile.checksum.hash,
78
+ start: options?.start ?? 0,
79
+ duration: options?.duration ?? media.duration
80
+ }
81
+
82
+ return item
83
+ }
84
+
85
+ audio = (
86
+ media: Media,
87
+ options?: {
88
+ start?: number,
89
+ duration?: number
90
+ }): Item.Audio => {
91
+
92
+ if(!media.hasAudio)
93
+ throw new Error(`Audio clip error: media "${media.datafile.filename}" has no audio track.`)
94
+
95
+ const item: Item.Audio = {
96
+ kind: Kind.Audio,
97
+ id: this.#getId(),
98
+ mediaHash: media.datafile.checksum.hash,
99
+ start: options?.start ?? 0,
100
+ duration: options?.duration ?? media.duration
101
+ }
102
+
103
+ return item
104
+ }
44
105
 
45
106
  text = (content: string): Item.Text => ({
46
107
  id: this.#getId(),
47
- kind: Kind.Text,
48
108
  content,
109
+ kind: Kind.Text,
110
+ color: "#FFFFF"
111
+ })
112
+
113
+ gap = (duration: number): Item.Gap => ({
114
+ id: this.#getId(),
115
+ kind: Kind.Gap,
116
+ duration
49
117
  })
50
118
 
51
119
  transition = {
@@ -56,5 +124,27 @@ export class O {
56
124
  duration,
57
125
  }),
58
126
  }
127
+
128
+ transform = (options?: TransformOptions): Transform => {
129
+ const position: Vec2 = [
130
+ options?.position?.[0] ?? 0,
131
+ options?.position?.[1] ?? 0
132
+ ]
133
+ const scale: Vec2 = [
134
+ options?.scale?.[0] ?? 1,
135
+ options?.scale?.[1] ?? 1
136
+ ]
137
+ const rotation = options?.rotation ?? 0
138
+ return [position, scale, rotation]
139
+ }
140
+
141
+ addChildren(parent: Item.Stack | Item.Sequence, ...items: Item.Any[]) {
142
+ this.#mutate(state => {
143
+ const parentItem = state.items.find(({id}) => id === parent.id) as Item.Stack
144
+ parentItem.childrenIds.push(...items.map(item => item.id))
145
+ state.items.push(...items)
146
+ return state
147
+ })
148
+ }
59
149
  }
60
150
 
@@ -1,12 +1,14 @@
1
1
 
2
2
  import {Omni} from "./omni.js"
3
+ import {Driver} from "../../driver/driver.js"
3
4
  import {dummyData} from "../utils/dummy-data.js"
4
5
 
6
+ const driver = await Driver.setup()
5
7
  //
6
8
  // create an omni context
7
9
  //
8
10
 
9
- const omni = new Omni()
11
+ const omni = new Omni(driver)
10
12
 
11
13
  //
12
14
  // load in some media resources
@@ -22,10 +24,10 @@ const {mediaA, mediaB} = await omni.load({
22
24
  //
23
25
 
24
26
  const timeline = omni.timeline(o => o.sequence(
25
- o.clip(mediaA),
27
+ o.video(mediaA),
26
28
  o.transition.crossfade(600),
27
29
  o.stack(
28
- o.clip(mediaB),
30
+ o.video(mediaB),
29
31
  o.text("hello world"),
30
32
  ),
31
33
  ))
@@ -2,12 +2,20 @@
2
2
  import {O} from "./o.js"
3
3
  import {Item} from "../parts/item.js"
4
4
  import {Media} from "../parts/media.js"
5
- import {TimelineFile} from "../parts/basics.js"
5
+ import {Driver} from "../../driver/driver.js"
6
6
  import {Datafile} from "../utils/datafile.js"
7
+ import {TimelineFile} from "../parts/basics.js"
8
+ import {Export} from "../parts/compositor/export.js"
7
9
  import {ResourcePool} from "../parts/resource-pool.js"
10
+ import {RenderConfig} from "../../driver/fns/schematic.js"
8
11
 
9
12
  export class Omni {
10
13
  resources = new ResourcePool()
14
+ #export: Export
15
+
16
+ constructor(private driver: Driver) {
17
+ this.#export = new Export(driver)
18
+ }
11
19
 
12
20
  load = async<S extends Record<string, Promise<Datafile>>>(spec: S) => {
13
21
  return Object.fromEntries(await Promise.all(Object.entries(spec).map(
@@ -15,16 +23,23 @@ export class Omni {
15
23
  ))) as {[K in keyof S]: Media}
16
24
  }
17
25
 
18
- timeline = (fn: (o: O) => Item.Sequence): TimelineFile => {
19
- const o = new O()
20
- const sequence = fn(o)
21
- return {
22
- format: "timeline",
23
- info: "https://omniclip.app/",
24
- version: 0,
25
- root: o.register(sequence),
26
- items: o.items,
27
- }
26
+ timeline = (fn: (o: O) => Item.Any): TimelineFile => {
27
+ const o = new O({
28
+ project: {
29
+ format: "timeline",
30
+ info: "https://omniclip.app/",
31
+ version: 0,
32
+ items: [],
33
+ rootId: 0
34
+ }
35
+ })
36
+ const root = fn(o)
37
+ o.state.project.rootId = root.id
38
+ return o.state.project
39
+ }
40
+
41
+ render = async (timeline: TimelineFile, config: RenderConfig) => {
42
+ await this.#export.render(timeline)
28
43
  }
29
44
  }
30
45
 
@@ -0,0 +1,29 @@
1
+ export type Interpolation = "linear" | "catmullRom"
2
+ export type Keyframe<Value = number> = [time: number, value: Value]
3
+ export type Keyframes<Value = number> = Keyframe<Value>[]
4
+ export type Vec2 = [x: number, y: number]
5
+ export type Transform = [position: Vec2, scale: Vec2, rotation: number]
6
+
7
+ export type TrackVec2 = {
8
+ x: Keyframes
9
+ y: Keyframes
10
+ }
11
+
12
+ export type Anim<T> = {
13
+ terp: Interpolation
14
+ track: T
15
+ }
16
+
17
+ export type Animations = Anim<TrackTransform>
18
+
19
+ export type TrackTransform = {
20
+ position: TrackVec2
21
+ scale: TrackVec2
22
+ rotation: Keyframes
23
+ }
24
+
25
+ export type TransformOptions = {
26
+ position?: Vec2
27
+ scale?: Vec2
28
+ rotation?: number
29
+ }
@@ -0,0 +1,15 @@
1
+ export class AudioStream {
2
+ constructor(private reader: ReadableStreamDefaultReader<AudioData>) {}
3
+
4
+ async *stream(): AsyncGenerator<AudioData> {
5
+ while (true) {
6
+ const {done, value: hit} = await this.reader.read()
7
+ if (done) {
8
+ break
9
+ }
10
+ yield hit
11
+ }
12
+ }
13
+
14
+ cancel = async () => await this.reader.cancel()
15
+ }
@@ -10,7 +10,8 @@ export class Checksum {
10
10
  ) {}
11
11
 
12
12
  static async make(data: Uint8Array) {
13
- const bytes = new Uint8Array(await crypto.subtle.digest("SHA-256", data))
13
+ const data2 = new Uint8Array(data)
14
+ const bytes = new Uint8Array(await crypto.subtle.digest("SHA-256", data2))
14
15
  const hash = Hex.fromBytes(bytes)
15
16
  const nickname = Thumbprint.sigil.fromBytes(bytes)
16
17
  return new this(data, bytes, hash, nickname)
@@ -0,0 +1,33 @@
1
+ import {Matrix} from "pixi.js"
2
+ import {Transform} from "../types.js"
3
+
4
+ export const transformToMat6 = (t: Transform): Mat6 => {
5
+ const [pos, scl, rotDeg] = t
6
+ const [x, y] = pos
7
+ const [sx, sy] = scl
8
+ const r = rotDeg * Math.PI / 180
9
+ const cos = Math.cos(r)
10
+ const sin = Math.sin(r)
11
+ return [cos * sx, sin * sx, -sin * sy, cos * sy, x, y]
12
+ }
13
+
14
+ export const mat6ToMatrix = ([a, b, c, d, tx, ty]: Mat6): Matrix =>
15
+ new Matrix(a, b, c, d, tx, ty)
16
+
17
+ export const transformToMatrix = (t: Transform) => mat6ToMatrix(transformToMat6(t))
18
+
19
+ export const mul6 = (local: Mat6, parent: Mat6): Mat6 => {
20
+ const [a1, b1, c1, d1, tx1, ty1] = local
21
+ const [a2, b2, c2, d2, tx2, ty2] = parent
22
+ return [
23
+ a1 * a2 + c1 * b2,
24
+ b1 * a2 + d1 * b2,
25
+ a1 * c2 + c1 * d2,
26
+ b1 * c2 + d1 * d2,
27
+ a1 * tx2 + c1 * ty2 + tx1,
28
+ b1 * tx2 + d1 * ty2 + ty1
29
+ ]
30
+ }
31
+
32
+ export const I6: Mat6 = [1, 0, 0, 1, 0, 0]
33
+ export type Mat6 = [a: number, b: number, c: number, d: number, tx: number, ty: number]
@@ -0,0 +1,40 @@
1
+ /**
2
+ * A stateful, forward-only frame cursor for a single clip instance.
3
+ * It efficiently reads a video stream to find the frame nearest to a target timestamp.
4
+ */
5
+
6
+ export class VideoCursor {
7
+ constructor(private reader: ReadableStreamDefaultReader<VideoFrame>) {}
8
+
9
+ async atOrNear(targetUs: number): Promise<VideoFrame | undefined> {
10
+ let prev: VideoFrame | null = null
11
+ while (true) {
12
+ const {done, value: hit} = await this.reader.read()
13
+
14
+ if (done) {
15
+ const out = prev ? new VideoFrame(prev) : undefined
16
+ prev?.close()
17
+ return out
18
+ }
19
+
20
+ const hitUs = hit.timestamp ?? 0
21
+ if (hitUs >= targetUs) {
22
+ const prevUs = prev?.timestamp ?? Number.NEGATIVE_INFINITY
23
+ const usePrev = !!prev && Math.abs(prevUs - targetUs) < Math.abs(hitUs - targetUs)
24
+
25
+ const chosen = usePrev ? prev! : hit
26
+ const other = usePrev ? hit : prev
27
+
28
+ const copy = new VideoFrame(chosen)
29
+ chosen.close()
30
+ other?.close()
31
+ return copy
32
+ }
33
+
34
+ prev?.close()
35
+ prev = hit
36
+ }
37
+ }
38
+
39
+ cancel = async () => await this.reader.cancel()
40
+ }
package/x/context.d.ts CHANGED
@@ -1,4 +1 @@
1
- import { Driver } from "./driver/driver.js";
2
- export declare const context: {
3
- driver: Promise<Driver>;
4
- };
1
+ export {};
package/x/context.js CHANGED
@@ -1,6 +1,2 @@
1
- import { Driver } from "./driver/driver.js";
2
- const workerUrl = new URL("../driver/driver.worker.bundle.js", import.meta.url);
3
- export const context = {
4
- driver: Driver.setup({ workerUrl })
5
- };
1
+ export {};
6
2
  //# sourceMappingURL=context.js.map
package/x/context.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"context.js","sourceRoot":"","sources":["../s/context.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,oBAAoB,CAAA;AAEzC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,mCAAmC,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AAE/E,MAAM,CAAC,MAAM,OAAO,GAAG;IACtB,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAC,SAAS,EAAC,CAAC;CACjC,CAAA"}
1
+ {"version":3,"file":"context.js","sourceRoot":"","sources":["../s/context.ts"],"names":[],"mappings":""}
@@ -1,14 +1,28 @@
1
- import { context } from "../context.js";
1
+ import { Driver } from "../driver/driver.js";
2
2
  import { waveformTest } from "./routines/waveform-test.js";
3
3
  import { filmstripTest } from "./routines/filmstrip-test.js";
4
4
  import { setupTranscodeTest } from "./routines/transcode-test.js";
5
- const driver = await context.driver;
5
+ import { Datafile, Omni, VideoPlayer } from "../timeline/index.js";
6
+ const driver = await Driver.setup();
6
7
  const results = document.querySelector(".results");
7
8
  const fetchButton = document.querySelector(".fetch");
8
9
  const importButton = document.querySelector(".import");
10
+ const playButton = document.querySelector(".play");
11
+ const stopButton = document.querySelector(".stop");
9
12
  fetchButton?.addEventListener("click", startDemoFetch);
10
13
  importButton?.addEventListener("click", startDemoImport);
11
- waveformTest();
14
+ const omni = new Omni(driver);
15
+ const file = await fetch("/assets/temp/gl.mp4");
16
+ const buffer = await file.arrayBuffer();
17
+ const uint = new Uint8Array(buffer);
18
+ const { videoA } = await omni.load({ videoA: Datafile.make(uint) });
19
+ const timeline = omni.timeline(o => o.sequence(o.stack(o.video(videoA, { duration: 5000 }), o.audio(videoA, { duration: 8000 })), o.video(videoA, { duration: 7000 })));
20
+ const player = await VideoPlayer.create(driver, timeline);
21
+ document.body.appendChild(player.canvas);
22
+ playButton.addEventListener("click", () => player.play());
23
+ stopButton.addEventListener("click", () => player.pause());
24
+ waveformTest(driver);
25
+ // const transcriber = await transcriberTest(driver)
12
26
  // hello world test
13
27
  {
14
28
  await driver.thread.work.hello();
@@ -20,9 +34,11 @@ waveformTest();
20
34
  // transcoding tests
21
35
  async function startDemoImport() {
22
36
  const [fileHandle] = await window.showOpenFilePicker();
23
- const transcode = setupTranscodeTest(driver, fileHandle);
24
- await filmstripTest(fileHandle);
37
+ const file = await fileHandle.getFile();
38
+ const transcode = setupTranscodeTest(driver, file);
39
+ await filmstripTest(file);
25
40
  run(transcode, fileHandle.name);
41
+ // await transcriber.transcribe(file)
26
42
  }
27
43
  async function startDemoFetch() {
28
44
  // which videos to run tests on
@@ -1 +1 @@
1
- {"version":3,"file":"demo.bundle.js","sourceRoot":"","sources":["../../s/demo/demo.bundle.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,OAAO,EAAC,MAAM,eAAe,CAAA;AACrC,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAA;AACxD,OAAO,EAAC,aAAa,EAAC,MAAM,8BAA8B,CAAA;AAC1D,OAAO,EAAC,kBAAkB,EAAC,MAAM,8BAA8B,CAAA;AAE/D,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAA;AACnC,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAE,CAAA;AAEnD,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAA;AACpD,MAAM,YAAY,GAAG,QAAQ,CAAC,aAAa,CAAC,SAAS,CAAsB,CAAA;AAE3E,WAAW,EAAE,gBAAgB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAA;AACtD,YAAY,EAAE,gBAAgB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;AAExD,YAAY,EAAE,CAAA;AAEd,mBAAmB;AACnB,CAAC;IACA,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,CAAA;IAChC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA;;QACxD,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAA;AACrD,CAAC;AAED,oBAAoB;AACpB,KAAK,UAAU,eAAe;IAE7B,MAAM,CAAC,UAAU,CAAC,GAAG,MAAM,MAAM,CAAC,kBAAkB,EAAE,CAAA;IACtD,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;IACxD,MAAM,aAAa,CAAC,UAAU,CAAC,CAAA;IAC/B,GAAG,CAAC,SAAS,EAAE,UAAU,CAAC,IAAI,CAAC,CAAA;AAChC,CAAC;AAED,KAAK,UAAU,cAAc;IAG5B,+BAA+B;IAC/B,MAAM,MAAM,GAAG;QACd,qBAAqB;KACrB,CAAA;IAED,gCAAgC;IAChC,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAA;QACnE,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;IACpB,CAAC;AACF,CAAC;AAED,KAAK,UAAU,GAAG,CAAC,SAAgD,EAAE,KAAa;IACjF,oBAAoB;IACpB,MAAM,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAA;IACzC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IAEnB,kBAAkB;IAClB,MAAM,CAAC,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAA;IACrC,CAAC,CAAC,WAAW,GAAG,KAAK,CAAA;IACrB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;IAEb,wBAAwB;IACxB,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IAE5B,eAAe;IACf,MAAM,SAAS,CAAC,GAAG,EAAE,CAAA;AACtB,CAAC"}
1
+ {"version":3,"file":"demo.bundle.js","sourceRoot":"","sources":["../../s/demo/demo.bundle.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,MAAM,EAAC,MAAM,qBAAqB,CAAA;AAC1C,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAA;AACxD,OAAO,EAAC,aAAa,EAAC,MAAM,8BAA8B,CAAA;AAE1D,OAAO,EAAC,kBAAkB,EAAC,MAAM,8BAA8B,CAAA;AAC/D,OAAO,EAAC,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAC,MAAM,sBAAsB,CAAA;AAEhE,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;AACnC,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAE,CAAA;AAEnD,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAA;AACpD,MAAM,YAAY,GAAG,QAAQ,CAAC,aAAa,CAAC,SAAS,CAAsB,CAAA;AAE3E,MAAM,UAAU,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAsB,CAAA;AACvE,MAAM,UAAU,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAsB,CAAA;AAEvE,WAAW,EAAE,gBAAgB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAA;AACtD,YAAY,EAAE,gBAAgB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;AAExD,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,CAAA;AAC7B,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,qBAAqB,CAAC,CAAA;AAC/C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAA;AACvC,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAA;AAEnC,MAAM,EAAC,MAAM,EAAC,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,EAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAC,CAAC,CAAA;AAC/D,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAClC,CAAC,CAAC,QAAQ,CACV,CAAC,CAAC,KAAK,CACN,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,EAAC,QAAQ,EAAE,IAAI,EAAC,CAAC,EACjC,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,EAAC,QAAQ,EAAE,IAAI,EAAC,CAAC,CACjC,EACD,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,EAAC,QAAQ,EAAE,IAAI,EAAC,CAAC,CACjC,CAAC,CAAA;AAEF,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;AACzD,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;AAExC,UAAU,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAA;AACzD,UAAU,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAA;AAE1D,YAAY,CAAC,MAAM,CAAC,CAAA;AACpB,oDAAoD;AAEpD,mBAAmB;AACnB,CAAC;IACA,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,CAAA;IAChC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA;;QACxD,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAA;AACrD,CAAC;AAED,oBAAoB;AACpB,KAAK,UAAU,eAAe;IAE7B,MAAM,CAAC,UAAU,CAAC,GAAG,MAAM,MAAM,CAAC,kBAAkB,EAAE,CAAA;IACtD,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,OAAO,EAAE,CAAA;IACvC,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;IAClD,MAAM,aAAa,CAAC,IAAI,CAAC,CAAA;IACzB,GAAG,CAAC,SAAS,EAAE,UAAU,CAAC,IAAI,CAAC,CAAA;IAC/B,qCAAqC;AACtC,CAAC;AAED,KAAK,UAAU,cAAc;IAG5B,+BAA+B;IAC/B,MAAM,MAAM,GAAG;QACd,qBAAqB;KACrB,CAAA;IAED,gCAAgC;IAChC,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAA;QACnE,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;IACpB,CAAC;AACF,CAAC;AAED,KAAK,UAAU,GAAG,CAAC,SAAgD,EAAE,KAAa;IACjF,oBAAoB;IACpB,MAAM,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAA;IACzC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IAEnB,kBAAkB;IAClB,MAAM,CAAC,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAA;IACrC,CAAC,CAAC,WAAW,GAAG,KAAK,CAAA;IACrB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;IAEb,wBAAwB;IACxB,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IAE5B,eAAe;IACf,MAAM,SAAS,CAAC,GAAG,EAAE,CAAA;AACtB,CAAC"}