@omnimedia/omnitool 1.1.0-3 → 1.1.0-30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. package/package.json +15 -10
  2. package/s/context.ts +0 -7
  3. package/s/demo/demo.bundle.ts +39 -5
  4. package/s/demo/demo.css +5 -0
  5. package/s/demo/routines/filmstrip-test.ts +2 -2
  6. package/s/demo/routines/transcode-test.ts +8 -4
  7. package/s/demo/routines/transcriber-test.ts +34 -0
  8. package/s/demo/routines/transitions-test.ts +43 -0
  9. package/s/demo/routines/waveform-test.ts +3 -2
  10. package/s/driver/driver.ts +19 -11
  11. package/s/driver/fns/host.ts +7 -6
  12. package/s/driver/fns/schematic.ts +47 -24
  13. package/s/driver/fns/work.ts +165 -156
  14. package/s/driver/utils/load-decoder-source.ts +3 -4
  15. package/s/features/speech/transcribe/default-spec.ts +11 -0
  16. package/s/features/speech/transcribe/parts/load-pipe.ts +19 -0
  17. package/s/features/speech/transcribe/parts/prep-audio.ts +23 -0
  18. package/s/features/speech/transcribe/parts/transcribe.ts +70 -0
  19. package/s/features/speech/transcribe/transcriber.ts +46 -0
  20. package/s/features/speech/transcribe/types.ts +82 -0
  21. package/s/features/speech/transcribe/worker.bundle.ts +40 -0
  22. package/s/features/transition/parts/fragment.ts +24 -0
  23. package/s/features/transition/parts/types.ts +94 -0
  24. package/s/features/transition/parts/uniforms.ts +29 -0
  25. package/s/features/transition/parts/vertex.ts +31 -0
  26. package/s/features/transition/transition.ts +60 -0
  27. package/s/index.html.ts +6 -1
  28. package/s/timeline/index.ts +1 -0
  29. package/s/timeline/parts/basics.ts +1 -1
  30. package/s/timeline/parts/compositor/export.ts +77 -0
  31. package/s/timeline/parts/compositor/parts/html-tree.ts +37 -0
  32. package/s/timeline/parts/compositor/parts/schedulers.ts +94 -0
  33. package/s/timeline/parts/compositor/parts/tree-builder.ts +196 -0
  34. package/s/timeline/parts/compositor/parts/webcodecs-tree.ts +30 -0
  35. package/s/timeline/parts/compositor/playback.ts +94 -0
  36. package/s/timeline/parts/compositor/samplers/html.ts +115 -0
  37. package/s/timeline/parts/compositor/samplers/webcodecs.ts +61 -0
  38. package/s/timeline/parts/item.ts +48 -6
  39. package/s/timeline/parts/media.ts +21 -0
  40. package/s/timeline/parts/waveform.ts +3 -4
  41. package/s/timeline/sugar/builders.ts +102 -0
  42. package/s/timeline/sugar/o.ts +162 -36
  43. package/s/timeline/sugar/omni-test.ts +5 -3
  44. package/s/timeline/sugar/omni.ts +26 -11
  45. package/s/timeline/types.ts +29 -0
  46. package/s/timeline/utils/audio-stream.ts +15 -0
  47. package/s/timeline/utils/checksum.ts +2 -1
  48. package/s/timeline/utils/matrix.ts +33 -0
  49. package/s/timeline/utils/video-cursor.ts +40 -0
  50. package/x/context.d.ts +1 -4
  51. package/x/context.js +1 -5
  52. package/x/context.js.map +1 -1
  53. package/x/demo/demo.bundle.js +26 -5
  54. package/x/demo/demo.bundle.js.map +1 -1
  55. package/x/demo/demo.bundle.min.js +606 -36
  56. package/x/demo/demo.bundle.min.js.map +4 -4
  57. package/x/demo/demo.css +5 -0
  58. package/x/demo/routines/filmstrip-test.d.ts +1 -1
  59. package/x/demo/routines/filmstrip-test.js +2 -2
  60. package/x/demo/routines/filmstrip-test.js.map +1 -1
  61. package/x/demo/routines/transcode-test.js +8 -4
  62. package/x/demo/routines/transcode-test.js.map +1 -1
  63. package/x/demo/routines/transcriber-test.d.ts +4 -0
  64. package/x/demo/routines/transcriber-test.js +33 -0
  65. package/x/demo/routines/transcriber-test.js.map +1 -0
  66. package/x/demo/routines/transitions-test.d.ts +5 -0
  67. package/x/demo/routines/transitions-test.js +35 -0
  68. package/x/demo/routines/transitions-test.js.map +1 -0
  69. package/x/demo/routines/waveform-test.d.ts +2 -1
  70. package/x/demo/routines/waveform-test.js +2 -2
  71. package/x/demo/routines/waveform-test.js.map +1 -1
  72. package/x/driver/driver.d.ts +4 -6
  73. package/x/driver/driver.js +17 -10
  74. package/x/driver/driver.js.map +1 -1
  75. package/x/driver/driver.worker.bundle.min.js +2537 -148
  76. package/x/driver/driver.worker.bundle.min.js.map +4 -4
  77. package/x/driver/fns/host.d.ts +9 -2
  78. package/x/driver/fns/host.js +3 -3
  79. package/x/driver/fns/host.js.map +1 -1
  80. package/x/driver/fns/schematic.d.ts +41 -23
  81. package/x/driver/fns/work.d.ts +11 -4
  82. package/x/driver/fns/work.js +113 -107
  83. package/x/driver/fns/work.js.map +1 -1
  84. package/x/driver/utils/load-decoder-source.d.ts +2 -1
  85. package/x/driver/utils/load-decoder-source.js +2 -3
  86. package/x/driver/utils/load-decoder-source.js.map +1 -1
  87. package/x/features/speech/transcribe/default-spec.d.ts +2 -0
  88. package/x/features/speech/transcribe/default-spec.js +8 -0
  89. package/x/features/speech/transcribe/default-spec.js.map +1 -0
  90. package/x/features/speech/transcribe/parts/load-pipe.d.ts +2 -0
  91. package/x/features/speech/transcribe/parts/load-pipe.js +13 -0
  92. package/x/features/speech/transcribe/parts/load-pipe.js.map +1 -0
  93. package/x/features/speech/transcribe/parts/prep-audio.d.ts +5 -0
  94. package/x/features/speech/transcribe/parts/prep-audio.js +21 -0
  95. package/x/features/speech/transcribe/parts/prep-audio.js.map +1 -0
  96. package/x/features/speech/transcribe/parts/transcribe.d.ts +5 -0
  97. package/x/features/speech/transcribe/parts/transcribe.js +56 -0
  98. package/x/features/speech/transcribe/parts/transcribe.js.map +1 -0
  99. package/x/features/speech/transcribe/transcriber.d.ts +5 -0
  100. package/x/features/speech/transcribe/transcriber.js +33 -0
  101. package/x/features/speech/transcribe/transcriber.js.map +1 -0
  102. package/x/features/speech/transcribe/types.d.ts +66 -0
  103. package/x/features/speech/transcribe/types.js +2 -0
  104. package/x/features/speech/transcribe/types.js.map +1 -0
  105. package/x/features/speech/transcribe/worker.bundle.d.ts +1 -0
  106. package/x/features/speech/transcribe/worker.bundle.js +33 -0
  107. package/x/features/speech/transcribe/worker.bundle.js.map +1 -0
  108. package/x/features/speech/transcribe/worker.bundle.min.js +2916 -0
  109. package/x/features/speech/transcribe/worker.bundle.min.js.map +7 -0
  110. package/x/features/transition/parts/fragment.d.ts +1 -0
  111. package/x/features/transition/parts/fragment.js +25 -0
  112. package/x/features/transition/parts/fragment.js.map +1 -0
  113. package/x/features/transition/parts/types.d.ts +23 -0
  114. package/x/features/transition/parts/types.js +2 -0
  115. package/x/features/transition/parts/types.js.map +1 -0
  116. package/x/features/transition/parts/uniforms.d.ts +31 -0
  117. package/x/features/transition/parts/uniforms.js +27 -0
  118. package/x/features/transition/parts/uniforms.js.map +1 -0
  119. package/x/features/transition/parts/vertex.d.ts +1 -0
  120. package/x/features/transition/parts/vertex.js +32 -0
  121. package/x/features/transition/parts/vertex.js.map +1 -0
  122. package/x/features/transition/transition.d.ts +5 -0
  123. package/x/features/transition/transition.js +50 -0
  124. package/x/features/transition/transition.js.map +1 -0
  125. package/x/index.html +13 -3
  126. package/x/index.html.js +6 -1
  127. package/x/index.html.js.map +1 -1
  128. package/x/timeline/index.d.ts +1 -0
  129. package/x/timeline/index.js +1 -0
  130. package/x/timeline/index.js.map +1 -1
  131. package/x/timeline/parts/basics.d.ts +1 -1
  132. package/x/timeline/parts/compositor/export.d.ts +11 -0
  133. package/x/timeline/parts/compositor/export.js +64 -0
  134. package/x/timeline/parts/compositor/export.js.map +1 -0
  135. package/x/timeline/parts/compositor/parts/html-tree.d.ts +3 -0
  136. package/x/timeline/parts/compositor/parts/html-tree.js +40 -0
  137. package/x/timeline/parts/compositor/parts/html-tree.js.map +1 -0
  138. package/x/timeline/parts/compositor/parts/schedulers.d.ts +15 -0
  139. package/x/timeline/parts/compositor/parts/schedulers.js +69 -0
  140. package/x/timeline/parts/compositor/parts/schedulers.js.map +1 -0
  141. package/x/timeline/parts/compositor/parts/tree-builder.d.ts +37 -0
  142. package/x/timeline/parts/compositor/parts/tree-builder.js +160 -0
  143. package/x/timeline/parts/compositor/parts/tree-builder.js.map +1 -0
  144. package/x/timeline/parts/compositor/parts/webcodecs-tree.d.ts +3 -0
  145. package/x/timeline/parts/compositor/parts/webcodecs-tree.js +28 -0
  146. package/x/timeline/parts/compositor/parts/webcodecs-tree.js.map +1 -0
  147. package/x/timeline/parts/compositor/playback.d.ts +26 -0
  148. package/x/timeline/parts/compositor/playback.js +79 -0
  149. package/x/timeline/parts/compositor/playback.js.map +1 -0
  150. package/x/timeline/parts/compositor/samplers/html.d.ts +3 -0
  151. package/x/timeline/parts/compositor/samplers/html.js +106 -0
  152. package/x/timeline/parts/compositor/samplers/html.js.map +1 -0
  153. package/x/timeline/parts/compositor/samplers/webcodecs.d.ts +3 -0
  154. package/x/timeline/parts/compositor/samplers/webcodecs.js +52 -0
  155. package/x/timeline/parts/compositor/samplers/webcodecs.js.map +1 -0
  156. package/x/timeline/parts/item.d.ts +42 -8
  157. package/x/timeline/parts/item.js +7 -3
  158. package/x/timeline/parts/item.js.map +1 -1
  159. package/x/timeline/parts/media.d.ts +3 -0
  160. package/x/timeline/parts/media.js +17 -0
  161. package/x/timeline/parts/media.js.map +1 -1
  162. package/x/timeline/parts/waveform.d.ts +2 -1
  163. package/x/timeline/parts/waveform.js +2 -4
  164. package/x/timeline/parts/waveform.js.map +1 -1
  165. package/x/timeline/sugar/builders.d.ts +1 -0
  166. package/x/timeline/sugar/builders.js +104 -0
  167. package/x/timeline/sugar/builders.js.map +1 -0
  168. package/x/timeline/sugar/o.d.ts +27 -5
  169. package/x/timeline/sugar/o.js +135 -36
  170. package/x/timeline/sugar/o.js.map +1 -1
  171. package/x/timeline/sugar/omni-test.js +4 -2
  172. package/x/timeline/sugar/omni-test.js.map +1 -1
  173. package/x/timeline/sugar/omni.d.ts +8 -2
  174. package/x/timeline/sugar/omni.js +22 -9
  175. package/x/timeline/sugar/omni.js.map +1 -1
  176. package/x/timeline/types.d.ts +24 -0
  177. package/x/timeline/types.js +2 -0
  178. package/x/timeline/types.js.map +1 -0
  179. package/x/timeline/utils/audio-stream.d.ts +6 -0
  180. package/x/timeline/utils/audio-stream.js +17 -0
  181. package/x/timeline/utils/audio-stream.js.map +1 -0
  182. package/x/timeline/utils/checksum.js +2 -1
  183. package/x/timeline/utils/checksum.js.map +1 -1
  184. package/x/timeline/utils/matrix.d.ts +8 -0
  185. package/x/timeline/utils/matrix.js +26 -0
  186. package/x/timeline/utils/matrix.js.map +1 -0
  187. package/x/timeline/utils/video-cursor.d.ts +10 -0
  188. package/x/timeline/utils/video-cursor.js +36 -0
  189. package/x/timeline/utils/video-cursor.js.map +1 -0
@@ -0,0 +1,102 @@
1
+ // import {O} from "./o.js"
2
+ // import {Id} from "../parts/basics.js"
3
+ // import {Item} from "../parts/item.js"
4
+ //
5
+ // export class TimelineItem {
6
+ // public readonly id: Id
7
+ //
8
+ // constructor(public item: Item.Any) {
9
+ // this.id = item.id
10
+ // }
11
+ //
12
+ // toJSON() {
13
+ // return {
14
+ // ...this.item
15
+ // }
16
+ // }
17
+ // }
18
+ //
19
+ // abstract class VisualItem extends TimelineItem {
20
+ // abstract spatial(spatial: Spatial): TimelineItem
21
+ // }
22
+ //
23
+ // export class Stack extends VisualItem {
24
+ // constructor(private o: O, public item: Item.Stack) {
25
+ // super(item)
26
+ // }
27
+ //
28
+ // spatial(spatial: Spatial) {
29
+ // this.item.spatialId = spatial.item.id
30
+ // return this
31
+ // }
32
+ //
33
+ // addChildren(fn: (o: O) => TimelineItem | TimelineItem[]) {
34
+ // const result = fn(this.o)
35
+ // const items = Array.isArray(result) ? result : [result]
36
+ // this.item.childrenIds.push(...items.map(c => c.item.id))
37
+ // return this
38
+ // }
39
+ // }
40
+ //
41
+ // export class Spatial extends TimelineItem {
42
+ // constructor(public item: Item.Spatial) {super(item)}
43
+ // }
44
+ //
45
+ // export class Gap extends TimelineItem {
46
+ // constructor(public item: Item.Gap) {super(item)}
47
+ // }
48
+ //
49
+ // export class Audio extends TimelineItem {
50
+ // constructor(public item: Item.Audio) {super(item)}
51
+ // }
52
+ //
53
+ // export class Video extends VisualItem {
54
+ // constructor(public item: Item.Video) {
55
+ // super(item)
56
+ // }
57
+ //
58
+ // spatial(spatial: Spatial) {
59
+ // this.item.spatialId = spatial.item.id
60
+ // return this
61
+ // }
62
+ // }
63
+ //
64
+ // export class Text extends VisualItem {
65
+ // constructor(public item: Item.Text) {
66
+ // super(item)
67
+ // }
68
+ //
69
+ // color(color: string) {
70
+ // this.item.color = color
71
+ // return this
72
+ // }
73
+ //
74
+ // spatial(spatial: Spatial) {
75
+ // this.item.spatialId = spatial.item.id
76
+ // return this
77
+ // }
78
+ // }
79
+ //
80
+ // export class Sequence extends VisualItem {
81
+ // constructor(private o: O, public item: Item.Sequence) {
82
+ // super(item)
83
+ // }
84
+ //
85
+ // spatial(spatial: Spatial) {
86
+ // this.item.spatialId = spatial.item.id
87
+ // return this
88
+ // }
89
+ //
90
+ // addChildren(fn: (o: O) => TimelineItem | TimelineItem[]) {
91
+ // const result = fn(this.o)
92
+ // const items = Array.isArray(result) ? result : [result]
93
+ // this.item.childrenIds.push(...items.map(c => c.item.id))
94
+ // return this
95
+ // }
96
+ // }
97
+ //
98
+ // export class Transition extends TimelineItem {
99
+ // constructor(public item: Item.Transition) {
100
+ // super(item)
101
+ // }
102
+ // }
@@ -1,60 +1,186 @@
1
+ import {TextStyleOptions} from "pixi.js"
1
2
 
2
- import {MapG} from "@e280/stz"
3
- import {Id} from "../parts/basics.js"
4
3
  import {Media} from "../parts/media.js"
4
+ import {Id, TimelineFile} from "../parts/basics.js"
5
5
  import {Effect, Item, Kind} from "../parts/item.js"
6
+ import {Transform, TransformOptions, Vec2} from "../types.js"
6
7
 
7
8
  export class O {
8
9
  #nextId = 0
9
- #items = new MapG<Id, Item.Any>()
10
+
11
+ constructor(public state: {project: TimelineFile}) {}
12
+
13
+ require<T extends Item.Any>(id: Id | undefined) {
14
+ if (id === undefined)
15
+ return undefined
16
+ const item = this.state.project.items.find(item => item.id === id)
17
+ return item as T | undefined
18
+ }
10
19
 
11
20
  #getId() {
12
21
  return this.#nextId++
13
22
  }
14
23
 
24
+ #mutate(fn: (project: TimelineFile) => TimelineFile) {
25
+ this.state.project = fn(this.state.project)
26
+ }
27
+
15
28
  register(item: Item.Any) {
16
- if (!this.#items.has(item.id))
17
- this.#items.set(item.id, item)
18
- return item.id
29
+ this.#mutate(state => ({
30
+ ...state,
31
+ items: [...state.items, item]
32
+ }))
33
+ }
34
+
35
+ textStyle = (style: TextStyleOptions): Item.TextStyle => {
36
+ const item = {
37
+ id: this.#getId(),
38
+ kind: Kind.TextStyle,
39
+ style
40
+ } as Item.TextStyle
41
+ this.register(item)
42
+ return item
43
+ }
44
+
45
+ spatial = (transform: Transform): Item.Spatial => {
46
+ const item: Item.Spatial = {
47
+ id: this.#getId(),
48
+ kind: Kind.Spatial,
49
+ transform
50
+ }
51
+ this.register(item)
52
+ return item
53
+ }
54
+
55
+ sequence = (...items: Item.Any[]): Item.Any => {
56
+ const item = {
57
+ id: this.#getId(),
58
+ kind: Kind.Sequence,
59
+ childrenIds: items.map(item => item.id)
60
+ } as Item.Sequence
61
+ this.register(item)
62
+ return item
19
63
  }
20
64
 
21
- get items() {
22
- return [...this.#items.values()]
65
+ stack = (...items: Item.Any[]): Item.Any => {
66
+ const item = {
67
+ kind: Kind.Stack,
68
+ id: this.#getId(),
69
+ childrenIds: items.map(item => item.id)
70
+ } as Item.Stack
71
+ this.register(item)
72
+ return item
23
73
  }
24
74
 
25
- sequence = (...items: Item.Any[]): Item.Sequence => ({
26
- id: this.#getId(),
27
- kind: Kind.Sequence,
28
- children: items.map(item => this.register(item)),
29
- })
75
+ video = (
76
+ media: Media,
77
+ options?: {
78
+ start?: number,
79
+ duration?: number
80
+ }): Item.Video => {
30
81
 
31
- stack = (...items: Item.Any[]): Item.Stack => ({
32
- id: this.#getId(),
33
- kind: Kind.Stack,
34
- children: items.map(item => this.register(item)),
35
- })
82
+ if(!media.hasVideo)
83
+ throw new Error(`Video clip error: media "${media.datafile.filename}" has no video track.`)
36
84
 
37
- clip = (media: Media, start?: number, duration?: number): Item.Clip => ({
38
- id: this.#getId(),
39
- kind: Kind.Clip,
40
- mediaHash: media.datafile.checksum.hash,
41
- start: start ?? 0,
42
- duration: duration ?? media.duration,
43
- })
85
+ const item: Item.Video = {
86
+ kind: Kind.Video,
87
+ id: this.#getId(),
88
+ mediaHash: media.datafile.checksum.hash,
89
+ start: options?.start ?? 0,
90
+ duration: options?.duration ?? media.duration
91
+ }
92
+ this.register(item)
93
+ return item
94
+ }
44
95
 
45
- text = (content: string): Item.Text => ({
46
- id: this.#getId(),
47
- kind: Kind.Text,
48
- content,
49
- })
96
+ audio = (
97
+ media: Media,
98
+ options?: {
99
+ start?: number,
100
+ duration?: number
101
+ }): Item.Audio => {
50
102
 
51
- transition = {
52
- crossfade: (duration: number): Item.Transition => ({
103
+ if(!media.hasAudio)
104
+ throw new Error(`Audio clip error: media "${media.datafile.filename}" has no audio track.`)
105
+
106
+ const item: Item.Audio = {
107
+ kind: Kind.Audio,
108
+ id: this.#getId(),
109
+ mediaHash: media.datafile.checksum.hash,
110
+ start: options?.start ?? 0,
111
+ duration: options?.duration ?? media.duration
112
+ }
113
+ this.register(item)
114
+ return item
115
+ }
116
+
117
+ text = (content: string): Item.Text => {
118
+ const item = {
119
+ id: this.#getId(),
120
+ content,
121
+ kind: Kind.Text,
122
+ duration: 2000
123
+ } as Item.Text
124
+ this.register(item)
125
+ return item
126
+ }
127
+
128
+ gap = (duration: number): Item.Gap => {
129
+ const item = {
53
130
  id: this.#getId(),
54
- kind: Kind.Transition,
55
- effect: Effect.Crossfade,
56
- duration,
57
- }),
131
+ kind: Kind.Gap,
132
+ duration
133
+ } as Item.Gap
134
+ this.register(item)
135
+ return item
136
+ }
137
+
138
+ transition = {
139
+ crossfade: (duration: number): Item.Transition => {
140
+ const item = {
141
+ id: this.#getId(),
142
+ kind: Kind.Transition,
143
+ effect: Effect.Crossfade,
144
+ duration,
145
+ } as Item.Transition
146
+ this.register(item)
147
+ return item
148
+ },
149
+ }
150
+
151
+ transform = (options?: TransformOptions): Transform => {
152
+ const position: Vec2 = [
153
+ options?.position?.[0] ?? 0,
154
+ options?.position?.[1] ?? 0
155
+ ]
156
+ const scale: Vec2 = [
157
+ options?.scale?.[0] ?? 1,
158
+ options?.scale?.[1] ?? 1
159
+ ]
160
+ const rotation = options?.rotation ?? 0
161
+ return [position, scale, rotation]
162
+ }
163
+
164
+ addChildren(parent: Item.Stack | Item.Sequence, ...items: Item.Any[]) {
165
+ this.#mutate(state => {
166
+ const parentItem = state.items.find(({id}) => id === parent.id) as Item.Stack
167
+ parentItem.childrenIds.push(...items.map(item => item.id))
168
+ return state
169
+ })
170
+ }
171
+
172
+ set = <K extends Item.Any>(
173
+ id: Id,
174
+ partial: Partial<K>
175
+ ) => {
176
+ this.#mutate(project => ({
177
+ ...project,
178
+ items: project.items.map(item =>
179
+ item.id === id
180
+ ? { ...item, ...partial }
181
+ : item
182
+ )
183
+ }))
58
184
  }
59
185
  }
60
186
 
@@ -1,12 +1,14 @@
1
1
 
2
2
  import {Omni} from "./omni.js"
3
+ import {Driver} from "../../driver/driver.js"
3
4
  import {dummyData} from "../utils/dummy-data.js"
4
5
 
6
+ const driver = await Driver.setup()
5
7
  //
6
8
  // create an omni context
7
9
  //
8
10
 
9
- const omni = new Omni()
11
+ const omni = new Omni(driver)
10
12
 
11
13
  //
12
14
  // load in some media resources
@@ -22,10 +24,10 @@ const {mediaA, mediaB} = await omni.load({
22
24
  //
23
25
 
24
26
  const timeline = omni.timeline(o => o.sequence(
25
- o.clip(mediaA),
27
+ o.video(mediaA),
26
28
  o.transition.crossfade(600),
27
29
  o.stack(
28
- o.clip(mediaB),
30
+ o.video(mediaB),
29
31
  o.text("hello world"),
30
32
  ),
31
33
  ))
@@ -2,12 +2,20 @@
2
2
  import {O} from "./o.js"
3
3
  import {Item} from "../parts/item.js"
4
4
  import {Media} from "../parts/media.js"
5
- import {TimelineFile} from "../parts/basics.js"
5
+ import {Driver} from "../../driver/driver.js"
6
6
  import {Datafile} from "../utils/datafile.js"
7
+ import {TimelineFile} from "../parts/basics.js"
8
+ import {Export} from "../parts/compositor/export.js"
7
9
  import {ResourcePool} from "../parts/resource-pool.js"
10
+ import {RenderConfig} from "../../driver/fns/schematic.js"
8
11
 
9
12
  export class Omni {
10
13
  resources = new ResourcePool()
14
+ #export: Export
15
+
16
+ constructor(private driver: Driver) {
17
+ this.#export = new Export(driver)
18
+ }
11
19
 
12
20
  load = async<S extends Record<string, Promise<Datafile>>>(spec: S) => {
13
21
  return Object.fromEntries(await Promise.all(Object.entries(spec).map(
@@ -15,16 +23,23 @@ export class Omni {
15
23
  ))) as {[K in keyof S]: Media}
16
24
  }
17
25
 
18
- timeline = (fn: (o: O) => Item.Sequence): TimelineFile => {
19
- const o = new O()
20
- const sequence = fn(o)
21
- return {
22
- format: "timeline",
23
- info: "https://omniclip.app/",
24
- version: 0,
25
- root: o.register(sequence),
26
- items: o.items,
27
- }
26
+ timeline = (fn: (o: O) => Item.Any): TimelineFile => {
27
+ const o = new O({
28
+ project: {
29
+ format: "timeline",
30
+ info: "https://omniclip.app/",
31
+ version: 0,
32
+ items: [],
33
+ rootId: 0
34
+ }
35
+ })
36
+ const root = fn(o)
37
+ o.state.project.rootId = root.id
38
+ return o.state.project
39
+ }
40
+
41
+ render = async (timeline: TimelineFile, config: RenderConfig) => {
42
+ await this.#export.render(timeline)
28
43
  }
29
44
  }
30
45
 
@@ -0,0 +1,29 @@
1
+ export type Interpolation = "linear" | "catmullRom"
2
+ export type Keyframe<Value = number> = [time: number, value: Value]
3
+ export type Keyframes<Value = number> = Keyframe<Value>[]
4
+ export type Vec2 = [x: number, y: number]
5
+ export type Transform = [position: Vec2, scale: Vec2, rotation: number]
6
+
7
+ export type TrackVec2 = {
8
+ x: Keyframes
9
+ y: Keyframes
10
+ }
11
+
12
+ export type Anim<T> = {
13
+ terp: Interpolation
14
+ track: T
15
+ }
16
+
17
+ export type Animations = Anim<TrackTransform>
18
+
19
+ export type TrackTransform = {
20
+ position: TrackVec2
21
+ scale: TrackVec2
22
+ rotation: Keyframes
23
+ }
24
+
25
+ export type TransformOptions = {
26
+ position?: Vec2
27
+ scale?: Vec2
28
+ rotation?: number
29
+ }
@@ -0,0 +1,15 @@
1
+ export class AudioStream {
2
+ constructor(private reader: ReadableStreamDefaultReader<AudioData>) {}
3
+
4
+ async *stream(): AsyncGenerator<AudioData> {
5
+ while (true) {
6
+ const {done, value: hit} = await this.reader.read()
7
+ if (done) {
8
+ break
9
+ }
10
+ yield hit
11
+ }
12
+ }
13
+
14
+ cancel = async () => await this.reader.cancel()
15
+ }
@@ -10,7 +10,8 @@ export class Checksum {
10
10
  ) {}
11
11
 
12
12
  static async make(data: Uint8Array) {
13
- const bytes = new Uint8Array(await crypto.subtle.digest("SHA-256", data))
13
+ const data2 = new Uint8Array(data)
14
+ const bytes = new Uint8Array(await crypto.subtle.digest("SHA-256", data2))
14
15
  const hash = Hex.fromBytes(bytes)
15
16
  const nickname = Thumbprint.sigil.fromBytes(bytes)
16
17
  return new this(data, bytes, hash, nickname)
@@ -0,0 +1,33 @@
1
+ import {Matrix} from "pixi.js"
2
+ import {Transform} from "../types.js"
3
+
4
+ export const transformToMat6 = (t: Transform): Mat6 => {
5
+ const [pos, scl, rotDeg] = t
6
+ const [x, y] = pos
7
+ const [sx, sy] = scl
8
+ const r = rotDeg * Math.PI / 180
9
+ const cos = Math.cos(r)
10
+ const sin = Math.sin(r)
11
+ return [cos * sx, sin * sx, -sin * sy, cos * sy, x, y]
12
+ }
13
+
14
+ export const mat6ToMatrix = ([a, b, c, d, tx, ty]: Mat6): Matrix =>
15
+ new Matrix(a, b, c, d, tx, ty)
16
+
17
+ export const transformToMatrix = (t: Transform) => mat6ToMatrix(transformToMat6(t))
18
+
19
+ export const mul6 = (local: Mat6, parent: Mat6): Mat6 => {
20
+ const [a1, b1, c1, d1, tx1, ty1] = local
21
+ const [a2, b2, c2, d2, tx2, ty2] = parent
22
+ return [
23
+ a1 * a2 + c1 * b2,
24
+ b1 * a2 + d1 * b2,
25
+ a1 * c2 + c1 * d2,
26
+ b1 * c2 + d1 * d2,
27
+ a1 * tx2 + c1 * ty2 + tx1,
28
+ b1 * tx2 + d1 * ty2 + ty1
29
+ ]
30
+ }
31
+
32
+ export const I6: Mat6 = [1, 0, 0, 1, 0, 0]
33
+ export type Mat6 = [a: number, b: number, c: number, d: number, tx: number, ty: number]
@@ -0,0 +1,40 @@
1
+ /**
2
+ * A stateful, forward-only frame cursor for a single clip instance.
3
+ * It efficiently reads a video stream to find the frame nearest to a target timestamp.
4
+ */
5
+
6
+ export class VideoCursor {
7
+ constructor(private reader: ReadableStreamDefaultReader<VideoFrame>) {}
8
+
9
+ async atOrNear(targetUs: number): Promise<VideoFrame | undefined> {
10
+ let prev: VideoFrame | null = null
11
+ while (true) {
12
+ const {done, value: hit} = await this.reader.read()
13
+
14
+ if (done) {
15
+ const out = prev ? new VideoFrame(prev) : undefined
16
+ prev?.close()
17
+ return out
18
+ }
19
+
20
+ const hitUs = hit.timestamp ?? 0
21
+ if (hitUs >= targetUs) {
22
+ const prevUs = prev?.timestamp ?? Number.NEGATIVE_INFINITY
23
+ const usePrev = !!prev && Math.abs(prevUs - targetUs) < Math.abs(hitUs - targetUs)
24
+
25
+ const chosen = usePrev ? prev! : hit
26
+ const other = usePrev ? hit : prev
27
+
28
+ const copy = new VideoFrame(chosen)
29
+ chosen.close()
30
+ other?.close()
31
+ return copy
32
+ }
33
+
34
+ prev?.close()
35
+ prev = hit
36
+ }
37
+ }
38
+
39
+ cancel = async () => await this.reader.cancel()
40
+ }
package/x/context.d.ts CHANGED
@@ -1,4 +1 @@
1
- import { Driver } from "./driver/driver.js";
2
- export declare const context: {
3
- driver: Promise<Driver>;
4
- };
1
+ export {};
package/x/context.js CHANGED
@@ -1,6 +1,2 @@
1
- import { Driver } from "./driver/driver.js";
2
- const workerUrl = new URL("../driver/driver.worker.bundle.js", import.meta.url);
3
- export const context = {
4
- driver: Driver.setup({ workerUrl })
5
- };
1
+ export {};
6
2
  //# sourceMappingURL=context.js.map
package/x/context.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"context.js","sourceRoot":"","sources":["../s/context.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,oBAAoB,CAAA;AAEzC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,mCAAmC,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AAE/E,MAAM,CAAC,MAAM,OAAO,GAAG;IACtB,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAC,SAAS,EAAC,CAAC;CACjC,CAAA"}
1
+ {"version":3,"file":"context.js","sourceRoot":"","sources":["../s/context.ts"],"names":[],"mappings":""}
@@ -1,14 +1,33 @@
1
- import { context } from "../context.js";
1
+ import { Driver } from "../driver/driver.js";
2
2
  import { waveformTest } from "./routines/waveform-test.js";
3
3
  import { filmstripTest } from "./routines/filmstrip-test.js";
4
4
  import { setupTranscodeTest } from "./routines/transcode-test.js";
5
- const driver = await context.driver;
5
+ import { Datafile, Omni, VideoPlayer } from "../timeline/index.js";
6
+ const driver = await Driver.setup({ workerUrl: new URL("../driver/driver.worker.bundle.min.js", import.meta.url) });
6
7
  const results = document.querySelector(".results");
7
8
  const fetchButton = document.querySelector(".fetch");
8
9
  const importButton = document.querySelector(".import");
10
+ const playButton = document.querySelector(".play");
11
+ const stopButton = document.querySelector(".stop");
12
+ const seekButton = document.querySelector(".seek");
9
13
  fetchButton?.addEventListener("click", startDemoFetch);
10
14
  importButton?.addEventListener("click", startDemoImport);
11
- waveformTest();
15
+ const omni = new Omni(driver);
16
+ const file = await fetch("/assets/temp/gl.mp4");
17
+ const buffer = await file.arrayBuffer();
18
+ const uint = new Uint8Array(buffer);
19
+ const { videoA } = await omni.load({ videoA: Datafile.make(uint) });
20
+ const timeline = omni.timeline(o => o.sequence(o.stack(o.video(videoA, { duration: 5000 }), o.audio(videoA, { duration: 8000 })), o.video(videoA, { duration: 7000 })));
21
+ const player = await VideoPlayer.create(driver, timeline);
22
+ document.body.appendChild(player.canvas);
23
+ playButton.addEventListener("click", () => player.play());
24
+ stopButton.addEventListener("click", () => player.pause());
25
+ seekButton.addEventListener("change", async (e) => {
26
+ const target = e.target;
27
+ await player.seek(+target.value);
28
+ });
29
+ waveformTest(driver);
30
+ // const transcriber = await transcriberTest(driver)
12
31
  // hello world test
13
32
  {
14
33
  await driver.thread.work.hello();
@@ -20,9 +39,11 @@ waveformTest();
20
39
  // transcoding tests
21
40
  async function startDemoImport() {
22
41
  const [fileHandle] = await window.showOpenFilePicker();
23
- const transcode = setupTranscodeTest(driver, fileHandle);
24
- await filmstripTest(fileHandle);
42
+ const file = await fileHandle.getFile();
43
+ const transcode = setupTranscodeTest(driver, file);
44
+ await filmstripTest(file);
25
45
  run(transcode, fileHandle.name);
46
+ // await transcriber.transcribe(file)
26
47
  }
27
48
  async function startDemoFetch() {
28
49
  // which videos to run tests on
@@ -1 +1 @@
1
- {"version":3,"file":"demo.bundle.js","sourceRoot":"","sources":["../../s/demo/demo.bundle.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,OAAO,EAAC,MAAM,eAAe,CAAA;AACrC,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAA;AACxD,OAAO,EAAC,aAAa,EAAC,MAAM,8BAA8B,CAAA;AAC1D,OAAO,EAAC,kBAAkB,EAAC,MAAM,8BAA8B,CAAA;AAE/D,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAA;AACnC,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAE,CAAA;AAEnD,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAA;AACpD,MAAM,YAAY,GAAG,QAAQ,CAAC,aAAa,CAAC,SAAS,CAAsB,CAAA;AAE3E,WAAW,EAAE,gBAAgB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAA;AACtD,YAAY,EAAE,gBAAgB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;AAExD,YAAY,EAAE,CAAA;AAEd,mBAAmB;AACnB,CAAC;IACA,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,CAAA;IAChC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA;;QACxD,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAA;AACrD,CAAC;AAED,oBAAoB;AACpB,KAAK,UAAU,eAAe;IAE7B,MAAM,CAAC,UAAU,CAAC,GAAG,MAAM,MAAM,CAAC,kBAAkB,EAAE,CAAA;IACtD,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;IACxD,MAAM,aAAa,CAAC,UAAU,CAAC,CAAA;IAC/B,GAAG,CAAC,SAAS,EAAE,UAAU,CAAC,IAAI,CAAC,CAAA;AAChC,CAAC;AAED,KAAK,UAAU,cAAc;IAG5B,+BAA+B;IAC/B,MAAM,MAAM,GAAG;QACd,qBAAqB;KACrB,CAAA;IAED,gCAAgC;IAChC,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAA;QACnE,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;IACpB,CAAC;AACF,CAAC;AAED,KAAK,UAAU,GAAG,CAAC,SAAgD,EAAE,KAAa;IACjF,oBAAoB;IACpB,MAAM,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAA;IACzC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IAEnB,kBAAkB;IAClB,MAAM,CAAC,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAA;IACrC,CAAC,CAAC,WAAW,GAAG,KAAK,CAAA;IACrB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;IAEb,wBAAwB;IACxB,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IAE5B,eAAe;IACf,MAAM,SAAS,CAAC,GAAG,EAAE,CAAA;AACtB,CAAC"}
1
+ {"version":3,"file":"demo.bundle.js","sourceRoot":"","sources":["../../s/demo/demo.bundle.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,MAAM,EAAC,MAAM,qBAAqB,CAAA;AAC1C,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAA;AACxD,OAAO,EAAC,aAAa,EAAC,MAAM,8BAA8B,CAAA;AAE1D,OAAO,EAAC,kBAAkB,EAAC,MAAM,8BAA8B,CAAA;AAC/D,OAAO,EAAC,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAC,MAAM,sBAAsB,CAAA;AAEhE,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,EAAC,SAAS,EAAE,IAAI,GAAG,CAAC,uCAAuC,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAC,CAAC,CAAA;AACjH,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAE,CAAA;AAEnD,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAA;AACpD,MAAM,YAAY,GAAG,QAAQ,CAAC,aAAa,CAAC,SAAS,CAAsB,CAAA;AAE3E,MAAM,UAAU,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAsB,CAAA;AACvE,MAAM,UAAU,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAsB,CAAA;AACvE,MAAM,UAAU,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAsB,CAAA;AAEvE,WAAW,EAAE,gBAAgB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAA;AACtD,YAAY,EAAE,gBAAgB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;AAExD,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,CAAA;AAC7B,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,qBAAqB,CAAC,CAAA;AAC/C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAA;AACvC,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAA;AAEnC,MAAM,EAAC,MAAM,EAAC,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,EAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAC,CAAC,CAAA;AAC/D,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAClC,CAAC,CAAC,QAAQ,CACV,CAAC,CAAC,KAAK,CACN,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,EAAC,QAAQ,EAAE,IAAI,EAAC,CAAC,EACjC,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,EAAC,QAAQ,EAAE,IAAI,EAAC,CAAC,CACjC,EACD,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,EAAC,QAAQ,EAAE,IAAI,EAAC,CAAC,CACjC,CAAC,CAAA;AAEF,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;AACzD,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;AAExC,UAAU,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAA;AACzD,UAAU,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAA;AAC1D,UAAU,CAAC,gBAAgB,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAQ,EAAE,EAAE;IACxD,MAAM,MAAM,GAAG,CAAC,CAAC,MAA0B,CAAA;IAC3C,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AACjC,CAAC,CAAC,CAAA;AAEF,YAAY,CAAC,MAAM,CAAC,CAAA;AACpB,oDAAoD;AAEpD,mBAAmB;AACnB,CAAC;IACA,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,CAAA;IAChC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA;;QACxD,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAA;AACrD,CAAC;AAED,oBAAoB;AACpB,KAAK,UAAU,eAAe;IAE7B,MAAM,CAAC,UAAU,CAAC,GAAG,MAAM,MAAM,CAAC,kBAAkB,EAAE,CAAA;IACtD,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,OAAO,EAAE,CAAA;IACvC,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;IAClD,MAAM,aAAa,CAAC,IAAI,CAAC,CAAA;IACzB,GAAG,CAAC,SAAS,EAAE,UAAU,CAAC,IAAI,CAAC,CAAA;IAC/B,qCAAqC;AACtC,CAAC;AAED,KAAK,UAAU,cAAc;IAG5B,+BAA+B;IAC/B,MAAM,MAAM,GAAG;QACd,qBAAqB;KACrB,CAAA;IAED,gCAAgC;IAChC,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAA;QACnE,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;IACpB,CAAC;AACF,CAAC;AAED,KAAK,UAAU,GAAG,CAAC,SAAgD,EAAE,KAAa;IACjF,oBAAoB;IACpB,MAAM,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAA;IACzC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IAEnB,kBAAkB;IAClB,MAAM,CAAC,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAA;IACrC,CAAC,CAAC,WAAW,GAAG,KAAK,CAAA;IACrB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;IAEb,wBAAwB;IACxB,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IAE5B,eAAe;IACf,MAAM,SAAS,CAAC,GAAG,EAAE,CAAA;AACtB,CAAC"}