@dxos/pipeline-transcription 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/neutral/chunk-Y44GOMGN.mjs +22 -0
- package/dist/lib/neutral/chunk-Y44GOMGN.mjs.map +7 -0
- package/dist/lib/neutral/index.mjs +886 -0
- package/dist/lib/neutral/index.mjs.map +7 -0
- package/dist/lib/neutral/meta.json +1 -0
- package/dist/lib/neutral/testing/index.mjs +192 -0
- package/dist/lib/neutral/testing/index.mjs.map +7 -0
- package/dist/types/src/PipelineConfig.d.ts +49 -0
- package/dist/types/src/PipelineConfig.d.ts.map +1 -0
- package/dist/types/src/PipelineConfig.test.d.ts +2 -0
- package/dist/types/src/PipelineConfig.test.d.ts.map +1 -0
- package/dist/types/src/PipelineRuntime.d.ts +46 -0
- package/dist/types/src/PipelineRuntime.d.ts.map +1 -0
- package/dist/types/src/PipelineRuntime.test.d.ts +2 -0
- package/dist/types/src/PipelineRuntime.test.d.ts.map +1 -0
- package/dist/types/src/Stage.d.ts +80 -0
- package/dist/types/src/Stage.d.ts.map +1 -0
- package/dist/types/src/TranscriptEvent.d.ts +23 -0
- package/dist/types/src/TranscriptEvent.d.ts.map +1 -0
- package/dist/types/src/asr/audio-recorder.d.ts +47 -0
- package/dist/types/src/asr/audio-recorder.d.ts.map +1 -0
- package/dist/types/src/asr/index.d.ts +5 -0
- package/dist/types/src/asr/index.d.ts.map +1 -0
- package/dist/types/src/asr/pipeline.d.ts +42 -0
- package/dist/types/src/asr/pipeline.d.ts.map +1 -0
- package/dist/types/src/asr/sentence-buffer.d.ts +25 -0
- package/dist/types/src/asr/sentence-buffer.d.ts.map +1 -0
- package/dist/types/src/asr/sentence-buffer.test.d.ts +2 -0
- package/dist/types/src/asr/sentence-buffer.test.d.ts.map +1 -0
- package/dist/types/src/asr/transcriber.d.ts +115 -0
- package/dist/types/src/asr/transcriber.d.ts.map +1 -0
- package/dist/types/src/asr/transcriber.test.d.ts +2 -0
- package/dist/types/src/asr/transcriber.test.d.ts.map +1 -0
- package/dist/types/src/asr/util.d.ts +2 -0
- package/dist/types/src/asr/util.d.ts.map +1 -0
- package/dist/types/src/dispatch.d.ts +19 -0
- package/dist/types/src/dispatch.d.ts.map +1 -0
- package/dist/types/src/dispatch.test.d.ts +2 -0
- package/dist/types/src/dispatch.test.d.ts.map +1 -0
- package/dist/types/src/index.d.ts +11 -0
- package/dist/types/src/index.d.ts.map +1 -0
- package/dist/types/src/live.d.ts +24 -0
- package/dist/types/src/live.d.ts.map +1 -0
- package/dist/types/src/live.test.d.ts +2 -0
- package/dist/types/src/live.test.d.ts.map +1 -0
- package/dist/types/src/lookup.d.ts +35 -0
- package/dist/types/src/lookup.d.ts.map +1 -0
- package/dist/types/src/lookup.test.d.ts +2 -0
- package/dist/types/src/lookup.test.d.ts.map +1 -0
- package/dist/types/src/model-routing.d.ts +12 -0
- package/dist/types/src/model-routing.d.ts.map +1 -0
- package/dist/types/src/pipeline-integration.test.d.ts +2 -0
- package/dist/types/src/pipeline-integration.test.d.ts.map +1 -0
- package/dist/types/src/stages/correction-llm.d.ts +14 -0
- package/dist/types/src/stages/correction-llm.d.ts.map +1 -0
- package/dist/types/src/stages/correction-llm.test.d.ts +2 -0
- package/dist/types/src/stages/correction-llm.test.d.ts.map +1 -0
- package/dist/types/src/stages/correction.d.ts +29 -0
- package/dist/types/src/stages/correction.d.ts.map +1 -0
- package/dist/types/src/stages/correction.test.d.ts +2 -0
- package/dist/types/src/stages/correction.test.d.ts.map +1 -0
- package/dist/types/src/stages/diarization.d.ts +11 -0
- package/dist/types/src/stages/diarization.d.ts.map +1 -0
- package/dist/types/src/stages/extraction.d.ts +40 -0
- package/dist/types/src/stages/extraction.d.ts.map +1 -0
- package/dist/types/src/stages/extraction.test.d.ts +2 -0
- package/dist/types/src/stages/extraction.test.d.ts.map +1 -0
- package/dist/types/src/stages/index.d.ts +18 -0
- package/dist/types/src/stages/index.d.ts.map +1 -0
- package/dist/types/src/stages/summarization.d.ts +12 -0
- package/dist/types/src/stages/summarization.d.ts.map +1 -0
- package/dist/types/src/stages/translation.d.ts +11 -0
- package/dist/types/src/stages/translation.d.ts.map +1 -0
- package/dist/types/src/testing/conversation-parser.d.ts +26 -0
- package/dist/types/src/testing/conversation-parser.d.ts.map +1 -0
- package/dist/types/src/testing/conversation.test.d.ts +2 -0
- package/dist/types/src/testing/conversation.test.d.ts.map +1 -0
- package/dist/types/src/testing/fixtures.d.ts +6 -0
- package/dist/types/src/testing/fixtures.d.ts.map +1 -0
- package/dist/types/src/testing/index.d.ts +5 -0
- package/dist/types/src/testing/index.d.ts.map +1 -0
- package/dist/types/src/testing/scripted-source.d.ts +16 -0
- package/dist/types/src/testing/scripted-source.d.ts.map +1 -0
- package/dist/types/src/testing/stream-simulator.d.ts +38 -0
- package/dist/types/src/testing/stream-simulator.d.ts.map +1 -0
- package/dist/types/tsconfig.tsbuildinfo +1 -0
- package/package.json +61 -0
- package/src/PipelineConfig.test.ts +39 -0
- package/src/PipelineConfig.ts +71 -0
- package/src/PipelineRuntime.test.ts +74 -0
- package/src/PipelineRuntime.ts +192 -0
- package/src/Stage.ts +86 -0
- package/src/TranscriptEvent.ts +23 -0
- package/src/asr/audio-recorder.ts +56 -0
- package/src/asr/index.ts +8 -0
- package/src/asr/pipeline.ts +105 -0
- package/src/asr/sentence-buffer.test.ts +53 -0
- package/src/asr/sentence-buffer.ts +88 -0
- package/src/asr/transcriber.test.ts +104 -0
- package/src/asr/transcriber.ts +337 -0
- package/src/asr/util.ts +14 -0
- package/src/dispatch.test.ts +37 -0
- package/src/dispatch.ts +56 -0
- package/src/index.ts +14 -0
- package/src/live.test.ts +63 -0
- package/src/live.ts +56 -0
- package/src/lookup.test.ts +41 -0
- package/src/lookup.ts +60 -0
- package/src/model-routing.ts +25 -0
- package/src/pipeline-integration.test.ts +70 -0
- package/src/stages/correction-llm.test.ts +46 -0
- package/src/stages/correction-llm.ts +40 -0
- package/src/stages/correction.test.ts +26 -0
- package/src/stages/correction.ts +72 -0
- package/src/stages/diarization.ts +24 -0
- package/src/stages/extraction.test.ts +79 -0
- package/src/stages/extraction.ts +151 -0
- package/src/stages/index.ts +44 -0
- package/src/stages/summarization.ts +37 -0
- package/src/stages/translation.ts +26 -0
- package/src/testing/conversation-parser.ts +101 -0
- package/src/testing/conversation.test.ts +53 -0
- package/src/testing/conversation.txt +23 -0
- package/src/testing/fixtures.ts +18 -0
- package/src/testing/index.ts +8 -0
- package/src/testing/scripted-source.ts +35 -0
- package/src/testing/stream-simulator.ts +125 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"PipelineRuntime.test.d.ts","sourceRoot":"","sources":["../../../src/PipelineRuntime.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import type * as Effect from 'effect/Effect';
|
|
2
|
+
import { DXN, type Ref } from '@dxos/echo';
|
|
3
|
+
import { type ContentBlock } from '@dxos/types';
|
|
4
|
+
import { type EntityLookup } from './lookup';
|
|
5
|
+
/**
|
|
6
|
+
* When a stage's discrete computation fires, relative to the pipeline stream.
|
|
7
|
+
* - `per-block`: on every new transcript block.
|
|
8
|
+
* - `on-silence`: when the speaker pauses (silence edge).
|
|
9
|
+
* - `periodic`: on a timer tick.
|
|
10
|
+
*/
|
|
11
|
+
export type StageTrigger = 'per-block' | 'on-silence' | 'periodic';
|
|
12
|
+
/**
|
|
13
|
+
* Policy applied when a stage is triggered while a previous invocation is still in flight.
|
|
14
|
+
* - `latest-wins`: interrupt the in-flight invocation and start a new one over the latest window.
|
|
15
|
+
* - `skip-if-busy`: drop the new trigger and let the in-flight invocation complete.
|
|
16
|
+
*/
|
|
17
|
+
export type StageConcurrency = 'latest-wins' | 'skip-if-busy';
|
|
18
|
+
/** Fields of a transcript block that stages may patch. */
|
|
19
|
+
export type BlockPatch = Partial<{
|
|
20
|
+
corrected: string;
|
|
21
|
+
references: Ref.Ref<any>[];
|
|
22
|
+
candidates: ContentBlock.Candidate[];
|
|
23
|
+
translation: string;
|
|
24
|
+
}>;
|
|
25
|
+
/** A patch to a single block, keyed by its index within the window snapshot passed to the stage. */
|
|
26
|
+
export type BlockUpdate = {
|
|
27
|
+
index: number;
|
|
28
|
+
} & BlockPatch;
|
|
29
|
+
/** A patch to the `Transcript` object itself. */
|
|
30
|
+
export type TranscriptUpdate = Partial<{
|
|
31
|
+
summary: string;
|
|
32
|
+
summaryUpdatedAt: string;
|
|
33
|
+
resolvedReferents: {
|
|
34
|
+
surface: string;
|
|
35
|
+
referent: string;
|
|
36
|
+
ref?: Ref.Ref<any>;
|
|
37
|
+
}[];
|
|
38
|
+
}>;
|
|
39
|
+
/** The result of a stage invocation: a typed description of ECHO mutations. */
|
|
40
|
+
export type StageWrite = {
|
|
41
|
+
blockUpdates?: BlockUpdate[];
|
|
42
|
+
transcriptUpdate?: TranscriptUpdate;
|
|
43
|
+
};
|
|
44
|
+
export declare const StageWrite: Readonly<{
|
|
45
|
+
blocks: (blockUpdates: BlockUpdate[]) => StageWrite;
|
|
46
|
+
transcript: (transcriptUpdate: TranscriptUpdate) => StageWrite;
|
|
47
|
+
empty: () => StageWrite;
|
|
48
|
+
}>;
|
|
49
|
+
/** Context supplied to a stage's `run` by the runtime. */
|
|
50
|
+
export type StageContext = {
|
|
51
|
+
/** Resolves entity references for extraction. Backend-agnostic (full-text, vector, …). */
|
|
52
|
+
readonly lookup?: EntityLookup;
|
|
53
|
+
/** The model resolved for this stage by the runtime (config → stage → preset default). */
|
|
54
|
+
readonly model: DXN.DXN;
|
|
55
|
+
};
|
|
56
|
+
/**
|
|
57
|
+
* A pipeline stage: a discrete, repeatedly-invoked transform over a window of the transcript stream.
|
|
58
|
+
*
|
|
59
|
+
* The runtime — not the stage — owns the continuous stream; it builds `input` from a window snapshot
|
|
60
|
+
* (`select`) and invokes `run` once per trigger, enforcing `concurrency` and injecting `model`.
|
|
61
|
+
*
|
|
62
|
+
* LLM-backed stages delegate `run` to an `Operation` (typed I/O, memoized-LLM tests, standalone
|
|
63
|
+
* re-invocation). Pure stages implement `run` directly.
|
|
64
|
+
*/
|
|
65
|
+
export interface Stage<In = unknown, E = unknown> {
|
|
66
|
+
readonly id: string;
|
|
67
|
+
readonly trigger: StageTrigger;
|
|
68
|
+
/** Sliding context size in blocks. Absent → the runtime passes the full current window. */
|
|
69
|
+
readonly window?: {
|
|
70
|
+
blocks: number;
|
|
71
|
+
};
|
|
72
|
+
readonly concurrency: StageConcurrency;
|
|
73
|
+
/** Default model for the stage; overridden by per-stage config in the runtime. */
|
|
74
|
+
readonly model?: DXN.DXN;
|
|
75
|
+
/** Build the stage input from the current window snapshot. Defaults to `{ window }`. */
|
|
76
|
+
readonly select?: (window: ContentBlock.Transcript[]) => In;
|
|
77
|
+
/** The discrete per-trigger computation. */
|
|
78
|
+
run(input: In, ctx: StageContext): Effect.Effect<StageWrite, E>;
|
|
79
|
+
}
|
|
80
|
+
//# sourceMappingURL=Stage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Stage.d.ts","sourceRoot":"","sources":["../../../src/Stage.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,KAAK,MAAM,MAAM,eAAe,CAAC;AAE7C,OAAO,EAAE,GAAG,EAAE,KAAK,GAAG,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,UAAU,CAAC;AAE7C;;;;;GAKG;AACH,MAAM,MAAM,YAAY,GAAG,WAAW,GAAG,YAAY,GAAG,UAAU,CAAC;AAEnE;;;;GAIG;AACH,MAAM,MAAM,gBAAgB,GAAG,aAAa,GAAG,cAAc,CAAC;AAE9D,0DAA0D;AAC1D,MAAM,MAAM,UAAU,GAAG,OAAO,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;IAC3B,UAAU,EAAE,YAAY,CAAC,SAAS,EAAE,CAAC;IACrC,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC,CAAC;AAEH,oGAAoG;AACpG,MAAM,MAAM,WAAW,GAAG;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG,UAAU,CAAC;AAEzD,iDAAiD;AACjD,MAAM,MAAM,gBAAgB,GAAG,OAAO,CAAC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;KAAE,EAAE,CAAC;CAChF,CAAC,CAAC;AAEH,+EAA+E;AAC/E,MAAM,MAAM,UAAU,GAAG;IACvB,YAAY,CAAC,EAAE,WAAW,EAAE,CAAC;IAC7B,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;CACrC,CAAC;AAEF,eAAO,MAAM,UAAU;2BACE,WAAW,EAAE,KAAG,UAAU;mCAClB,gBAAgB,KAAG,UAAU;iBACjD,UAAU;EACrB,CAAC;AAEH,0DAA0D;AAC1D,MAAM,MAAM,YAAY,GAAG;IACzB,0FAA0F;IAC1F,QAAQ,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC;IAC/B,0FAA0F;IAC1F,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC;CACzB,CAAC;AAEF;;;;;;;;GAQG;AACH,MAAM,WAAW,KAAK,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO;IAC9C,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;IAC/B,2FAA2F;IAC3F,QAAQ,CAAC,MAAM,CAAC,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,gBAAgB,CAAC;IACvC,kFAAkF;IAClF,QAAQ,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC;IACzB,wFAAwF;IACxF,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,YAAY,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC;IAC5D,4CAA4C;IAC5C,GAAG,CAAC,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;CACjE"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { type ContentBlock } from '@dxos/types';
|
|
2
|
+
/**
|
|
3
|
+
* Events flowing through the pipeline stream. The runtime owns this continuous stream and slices
|
|
4
|
+
* it into bounded windows that drive discrete stage invocations.
|
|
5
|
+
*/
|
|
6
|
+
export type TranscriptEvent = {
|
|
7
|
+
kind: 'block';
|
|
8
|
+
block: ContentBlock.Transcript;
|
|
9
|
+
} | {
|
|
10
|
+
kind: 'silence';
|
|
11
|
+
sinceMs: number;
|
|
12
|
+
} | {
|
|
13
|
+
kind: 'tick';
|
|
14
|
+
};
|
|
15
|
+
export declare const TranscriptEvent: Readonly<{
|
|
16
|
+
/** A new transcript block was appended by the ASR source. */
|
|
17
|
+
block: (block: ContentBlock.Transcript) => TranscriptEvent;
|
|
18
|
+
/** The speaker has been silent for `sinceMs` milliseconds. */
|
|
19
|
+
silence: (sinceMs: number) => TranscriptEvent;
|
|
20
|
+
/** Periodic timer tick (drives periodic-trigger stages). */
|
|
21
|
+
tick: () => TranscriptEvent;
|
|
22
|
+
}>;
|
|
23
|
+
//# sourceMappingURL=TranscriptEvent.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"TranscriptEvent.d.ts","sourceRoot":"","sources":["../../../src/TranscriptEvent.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD;;;GAGG;AACH,MAAM,MAAM,eAAe,GACvB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,YAAY,CAAC,UAAU,CAAA;CAAE,GACjD;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACpC;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAErB,eAAO,MAAM,eAAe;IAC1B,6DAA6D;mBAC9C,YAAY,CAAC,UAAU,KAAG,eAAe;IACxD,8DAA8D;uBAC3C,MAAM,KAAG,eAAe;IAC3C,4DAA4D;gBAClD,eAAe;EACzB,CAAC"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
export type AudioChunk = {
|
|
2
|
+
/**
|
|
3
|
+
* Timestamp of the beginning of the chunk.
|
|
4
|
+
*/
|
|
5
|
+
timestamp: number;
|
|
6
|
+
/**
|
|
7
|
+
* 16-bit PCM data.
|
|
8
|
+
*/
|
|
9
|
+
data: Float64Array;
|
|
10
|
+
};
|
|
11
|
+
export type WavConfig = {
|
|
12
|
+
channels: number;
|
|
13
|
+
sampleRate: number;
|
|
14
|
+
bitDepthCode: string;
|
|
15
|
+
};
|
|
16
|
+
/**
|
|
17
|
+
* Records MediaStreamTrack to Uint8Array 16-bit PCM data.
|
|
18
|
+
* The data is passed to the onChunk callback with the timestamp of the chunk.
|
|
19
|
+
* All chunks are aligned and sequential.
|
|
20
|
+
* Time ─────────────────────────────────────────────────►
|
|
21
|
+
*
|
|
22
|
+
* Recording:
|
|
23
|
+
* t1 ┌──────┐
|
|
24
|
+
* t2 ┌──────┬──────┐
|
|
25
|
+
* t3 ┌──────┬──────┬──────┐
|
|
26
|
+
* t4 ┌──────┬──────┬──────┬──────┐
|
|
27
|
+
* └──────┴──────┴──────┴──────┘
|
|
28
|
+
* 1st 2nd 3rd 4th
|
|
29
|
+
* chunk chunk chunk chunk
|
|
30
|
+
*
|
|
31
|
+
*/
|
|
32
|
+
export interface AudioRecorder {
|
|
33
|
+
wavConfig: WavConfig;
|
|
34
|
+
/**
|
|
35
|
+
* Sets callback for receiving audio chunks.
|
|
36
|
+
*/
|
|
37
|
+
setOnChunk(onChunk: (chunk: AudioChunk) => void): void;
|
|
38
|
+
/**
|
|
39
|
+
* Starts the recorder.
|
|
40
|
+
*/
|
|
41
|
+
start(): Promise<void>;
|
|
42
|
+
/**
|
|
43
|
+
* Stops the recorder.
|
|
44
|
+
*/
|
|
45
|
+
stop(): Promise<void>;
|
|
46
|
+
}
|
|
47
|
+
//# sourceMappingURL=audio-recorder.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"audio-recorder.d.ts","sourceRoot":"","sources":["../../../../src/asr/audio-recorder.ts"],"names":[],"mappings":"AAIA,MAAM,MAAM,UAAU,GAAG;IACvB;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,IAAI,EAAE,YAAY,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,SAAS,CAAC;IAErB;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,GAAG,IAAI,CAAC;IAEvD;;OAEG;IACH,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvB;;OAEG;IACH,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACvB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/asr/index.ts"],"names":[],"mappings":"AAIA,cAAc,kBAAkB,CAAC;AACjC,cAAc,mBAAmB,CAAC;AAClC,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { type ContentBlock } from '@dxos/types';
|
|
2
|
+
import { type RunOptions } from '../PipelineRuntime';
|
|
3
|
+
import { type AudioRecorder } from './audio-recorder';
|
|
4
|
+
import { type TranscribeConfig, type TranscribeFn } from './transcriber';
|
|
5
|
+
export type AsrPipelineOptions = Omit<RunOptions, 'source'> & {
|
|
6
|
+
/** Audio source — browser `MediaStreamRecorder` or any `AudioRecorder` implementation. */
|
|
7
|
+
recorder: AudioRecorder;
|
|
8
|
+
/** Optional ASR transport override (default: `fetch` to the edge transcription service). */
|
|
9
|
+
transcribe?: TranscribeFn;
|
|
10
|
+
/** Transcriber chunking configuration. */
|
|
11
|
+
config: TranscribeConfig;
|
|
12
|
+
/**
|
|
13
|
+
* Re-segment the ASR output into complete sentences before it enters the pipeline, merging
|
|
14
|
+
* fragments that the transcriber cut mid-sentence at chunk boundaries. @default false
|
|
15
|
+
*/
|
|
16
|
+
segmentSentences?: boolean;
|
|
17
|
+
/** Optional raw-segment hook invoked before each block enters the pipeline (e.g. live display). */
|
|
18
|
+
onSegment?: (block: ContentBlock.Transcript) => void;
|
|
19
|
+
/** Silence (ms) reported on `end` so on-silence stages fire over the final window. */
|
|
20
|
+
drainSilenceMs?: number;
|
|
21
|
+
};
|
|
22
|
+
/** Imperative handle to a running ASR → pipeline session. */
|
|
23
|
+
export type AsrPipeline = {
|
|
24
|
+
/** Open the recorder + transcriber. */
|
|
25
|
+
open: () => Promise<void>;
|
|
26
|
+
/** Begin capturing audio for transcription. */
|
|
27
|
+
start: () => void;
|
|
28
|
+
/** Stop capturing (buffered audio is still flushed by `end`). */
|
|
29
|
+
stop: () => void;
|
|
30
|
+
/** Abort the in-flight transcription request. */
|
|
31
|
+
abort: () => void;
|
|
32
|
+
/** Flush buffered audio, signal silence, and drain the pipeline. */
|
|
33
|
+
end: () => Promise<void>;
|
|
34
|
+
};
|
|
35
|
+
/**
|
|
36
|
+
* The single end-to-end wiring: an ASR source ({@link Transcriber}) feeding the live orchestration
|
|
37
|
+
* engine ({@link runLivePipeline}). Every consumer (editor driver, message-list recording, stories)
|
|
38
|
+
* differs only by recorder, stages, and commit sink — this is the one place audio → text → stages →
|
|
39
|
+
* sink is assembled.
|
|
40
|
+
*/
|
|
41
|
+
export declare const runAsrPipeline: ({ recorder, transcribe, config, segmentSentences, onSegment, drainSilenceMs, ...runOptions }: AsrPipelineOptions) => AsrPipeline;
|
|
42
|
+
//# sourceMappingURL=pipeline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../../src/asr/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAGhD,OAAO,EAAE,KAAK,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEtD,OAAO,EAAE,KAAK,gBAAgB,EAAE,KAAK,YAAY,EAAe,MAAM,eAAe,CAAC;AAEtF,MAAM,MAAM,kBAAkB,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG;IAC5D,0FAA0F;IAC1F,QAAQ,EAAE,aAAa,CAAC;IACxB,4FAA4F;IAC5F,UAAU,CAAC,EAAE,YAAY,CAAC;IAC1B,0CAA0C;IAC1C,MAAM,EAAE,gBAAgB,CAAC;IACzB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,mGAAmG;IACnG,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,CAAC,UAAU,KAAK,IAAI,CAAC;IACrD,sFAAsF;IACtF,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,6DAA6D;AAC7D,MAAM,MAAM,WAAW,GAAG;IACxB,uCAAuC;IACvC,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1B,+CAA+C;IAC/C,KAAK,EAAE,MAAM,IAAI,CAAC;IAClB,iEAAiE;IACjE,IAAI,EAAE,MAAM,IAAI,CAAC;IACjB,iDAAiD;IACjD,KAAK,EAAE,MAAM,IAAI,CAAC;IAClB,oEAAoE;IACpE,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,cAAc,iGAQxB,kBAAkB,KAAG,WA8CvB,CAAC"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { type ContentBlock } from '@dxos/types';
|
|
2
|
+
/**
|
|
3
|
+
* Splits text into complete sentences plus a trailing incomplete remainder.
|
|
4
|
+
*
|
|
5
|
+
* A run of `.`/`!`/`?` followed by whitespace or end-of-string terminates a sentence; an ellipsis
|
|
6
|
+
* (`..`+) is treated as non-terminal so mid-thought pauses keep accumulating. Abbreviations are not
|
|
7
|
+
* special-cased — this is a deterministic heuristic, not a full sentence tokenizer.
|
|
8
|
+
*/
|
|
9
|
+
export declare const splitSentences: (text: string) => {
|
|
10
|
+
sentences: string[];
|
|
11
|
+
rest: string;
|
|
12
|
+
};
|
|
13
|
+
/**
|
|
14
|
+
* Accumulates ASR fragments (which cut mid-sentence at chunk boundaries) and re-emits complete,
|
|
15
|
+
* sentence-aligned transcript blocks. Each emitted block inherits the `started` timestamp of the
|
|
16
|
+
* fragment that began the sentence. `flush` emits whatever remains (e.g. on stop).
|
|
17
|
+
*/
|
|
18
|
+
export declare class SentenceBuffer {
|
|
19
|
+
#private;
|
|
20
|
+
/** Append a fragment; returns any complete sentences it produced. */
|
|
21
|
+
push(block: ContentBlock.Transcript): ContentBlock.Transcript[];
|
|
22
|
+
/** Emit any buffered text as a final sentence. */
|
|
23
|
+
flush(): ContentBlock.Transcript[];
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=sentence-buffer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sentence-buffer.d.ts","sourceRoot":"","sources":["../../../../src/asr/sentence-buffer.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD;;;;;;GAMG;AACH,eAAO,MAAM,cAAc,SAAU,MAAM,KAAG;IAAE,SAAS,EAAE,MAAM,EAAE,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CA0BhF,CAAC;AAEF;;;;GAIG;AACH,qBAAa,cAAc;;IAIzB,qEAAqE;IACrE,IAAI,CAAC,KAAK,EAAE,YAAY,CAAC,UAAU,GAAG,YAAY,CAAC,UAAU,EAAE,CAuB9D;IAED,kDAAkD;IAClD,KAAK,IAAI,YAAY,CAAC,UAAU,EAAE,CASjC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sentence-buffer.test.d.ts","sourceRoot":"","sources":["../../../../src/asr/sentence-buffer.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { type Context, Resource } from '@dxos/context';
|
|
2
|
+
import { type ContentBlock } from '@dxos/types';
|
|
3
|
+
import { type AudioChunk, type AudioRecorder } from './audio-recorder';
|
|
4
|
+
export type WhisperWord = {
|
|
5
|
+
word: string;
|
|
6
|
+
/**
|
|
7
|
+
* Time in seconds from the start of the audio.
|
|
8
|
+
*/
|
|
9
|
+
start: number;
|
|
10
|
+
/**
|
|
11
|
+
* Time in seconds from the start of the audio.
|
|
12
|
+
*/
|
|
13
|
+
end: number;
|
|
14
|
+
};
|
|
15
|
+
export type WhisperSegment = {
|
|
16
|
+
text: string;
|
|
17
|
+
/**
|
|
18
|
+
* Time in seconds from the start of the audio.
|
|
19
|
+
*/
|
|
20
|
+
start: number;
|
|
21
|
+
/**
|
|
22
|
+
* Time in seconds from the start of the audio.
|
|
23
|
+
*/
|
|
24
|
+
end: number;
|
|
25
|
+
/**
|
|
26
|
+
* Probability of no speech in the segment.
|
|
27
|
+
*/
|
|
28
|
+
no_speech_prob: number;
|
|
29
|
+
words: WhisperWord[];
|
|
30
|
+
};
|
|
31
|
+
export type TranscribeConfig = {
|
|
32
|
+
/**
|
|
33
|
+
* Number of chunks to transcribe automatically after.
|
|
34
|
+
*/
|
|
35
|
+
transcribeAfterChunksAmount: number;
|
|
36
|
+
/**
|
|
37
|
+
* Number of chunks to save before the user starts speaking.
|
|
38
|
+
* This is needed to catch the beginning of the user's speech.
|
|
39
|
+
*/
|
|
40
|
+
prefixBufferChunksAmount: number;
|
|
41
|
+
/**
|
|
42
|
+
* Override the transcription endpoint base URL.
|
|
43
|
+
* Defaults to the DXOS calls service.
|
|
44
|
+
*/
|
|
45
|
+
endpoint?: string;
|
|
46
|
+
};
|
|
47
|
+
/**
|
|
48
|
+
* Function that converts a base64-encoded WAV payload into Whisper segments.
|
|
49
|
+
* Allows callers to swap in alternative providers or mock the transport.
|
|
50
|
+
*/
|
|
51
|
+
export type TranscribeFn = (audio: string) => Promise<WhisperSegment[]>;
|
|
52
|
+
export type TranscriberProps = {
|
|
53
|
+
config: TranscribeConfig;
|
|
54
|
+
recorder: AudioRecorder;
|
|
55
|
+
/**
|
|
56
|
+
* Optional override of the transcription transport. When provided, supersedes `config.endpoint`.
|
|
57
|
+
*/
|
|
58
|
+
transcribe?: TranscribeFn;
|
|
59
|
+
/**
|
|
60
|
+
* Callback to handle the transcribed segments, after all segment transformers are applied.
|
|
61
|
+
* @param segments - The transcribed segments.
|
|
62
|
+
*/
|
|
63
|
+
onSegments: (segments: ContentBlock.Transcript[]) => Promise<void>;
|
|
64
|
+
};
|
|
65
|
+
/**
|
|
66
|
+
* Handles transcription of audio chunks.
|
|
67
|
+
* If user is not speaking, the last `minChunksAmount` chunks are saved and transcribed.
|
|
68
|
+
* If user is speaking, the chunks are added to the buffer until the user is done talking.
|
|
69
|
+
*/
|
|
70
|
+
export declare class Transcriber extends Resource {
|
|
71
|
+
private _audioChunks;
|
|
72
|
+
private _lastTimestamp;
|
|
73
|
+
private readonly _openTrigger;
|
|
74
|
+
private readonly _config;
|
|
75
|
+
private readonly _recorder;
|
|
76
|
+
private readonly _onSegments;
|
|
77
|
+
private readonly _transcribeFn?;
|
|
78
|
+
private _recording;
|
|
79
|
+
private _transcribeTask?;
|
|
80
|
+
/** Aborts the in-flight transcription request (e.g. when the user cancels the drain). */
|
|
81
|
+
private _transcribeAbort?;
|
|
82
|
+
constructor({ config, recorder, onSegments, transcribe }: TranscriberProps);
|
|
83
|
+
protected _open(ctx: Context): Promise<void>;
|
|
84
|
+
protected _close(): Promise<void>;
|
|
85
|
+
/** Abort the in-flight transcription request (e.g. user cancelled the drain). */
|
|
86
|
+
abort(): void;
|
|
87
|
+
startChunksRecording(): void;
|
|
88
|
+
stopChunksRecording(): void;
|
|
89
|
+
/**
|
|
90
|
+
* Stop recording and transcribe any buffered audio, resolving once the final segments have been
|
|
91
|
+
* delivered via `onSegments`. Used to drain the pipeline when recording stops so the tail of speech
|
|
92
|
+
* (and its downstream enrichment) is not lost.
|
|
93
|
+
*/
|
|
94
|
+
flush(): Promise<void>;
|
|
95
|
+
private _saveAudioChunk;
|
|
96
|
+
private _transcribe;
|
|
97
|
+
private _dropOldChunks;
|
|
98
|
+
private _mergeAudioChunks;
|
|
99
|
+
private _fetchTranscription;
|
|
100
|
+
private _alignSegments;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Pure helper that filters and absolute-timestamps Whisper segments against a chunk timeline.
|
|
104
|
+
* Exported for testing.
|
|
105
|
+
*
|
|
106
|
+
* @param segments - Raw segments returned by the Whisper service for `originalChunks`.
|
|
107
|
+
* @param originalChunks - Chunks fed into the transcription request (used for the zero timestamp).
|
|
108
|
+
* @param lastTimestamp - Most recent absolute timestamp already emitted; segments whose end
|
|
109
|
+
* falls strictly before this point are deduped out, as are individual words whose start does.
|
|
110
|
+
*/
|
|
111
|
+
export declare const alignWhisperSegments: (segments: WhisperSegment[], originalChunks: AudioChunk[], lastTimestamp: number) => {
|
|
112
|
+
transcripts: ContentBlock.Transcript[];
|
|
113
|
+
lastTimestamp?: number;
|
|
114
|
+
};
|
|
115
|
+
//# sourceMappingURL=transcriber.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transcriber.d.ts","sourceRoot":"","sources":["../../../../src/asr/transcriber.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,KAAK,OAAO,EAAkB,QAAQ,EAAE,MAAM,eAAe,CAAC;AAGvE,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAAE,KAAK,UAAU,EAAE,KAAK,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGvE,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;CACb,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IAEZ;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB,KAAK,EAAE,WAAW,EAAE,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;OAEG;IACH,2BAA2B,EAAE,MAAM,CAAC;IAEpC;;;OAGG;IACH,wBAAwB,EAAE,MAAM,CAAC;IAEjC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;AAExE,MAAM,MAAM,gBAAgB,GAAG;IAC7B,MAAM,EAAE,gBAAgB,CAAC;IACzB,QAAQ,EAAE,aAAa,CAAC;IACxB;;OAEG;IACH,UAAU,CAAC,EAAE,YAAY,CAAC;IAC1B;;;OAGG;IACH,UAAU,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC,UAAU,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACpE,CAAC;AAEF;;;;GAIG;AACH,qBAAa,WAAY,SAAQ,QAAQ;IACvC,OAAO,CAAC,YAAY,CAAoB;IACxC,OAAO,CAAC,cAAc,CAAK;IAE3B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAoC;IAEjE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgB;IAC1C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAiC;IAC7D,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAe;IAE9C,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,eAAe,CAAC,CAA2B;IACnD,yFAAyF;IACzF,OAAO,CAAC,gBAAgB,CAAC,CAA8B;IAEvD,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,EAAE,gBAAgB,EAMzE;IAED,UAAyB,KAAK,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAM1D;IAED,UAAyB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAM/C;IAED,iFAAiF;IACjF,KAAK,IAAI,IAAI,CAEZ;IAED,oBAAoB,IAAI,IAAI,CAO3B;IAED,mBAAmB,IAAI,IAAI,CAQ1B;IAED;;;;OAIG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAO3B;IAED,OAAO,CAAC,eAAe;YAmBT,WAAW;IAoBzB,OAAO,CAAC,cAAc;YAOR,iBAAiB;YAejB,mBAAmB;IA+CjC,OAAO,CAAC,cAAc;CAQvB;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,oBAAoB,aACrB,cAAc,EAAE,kBACV,UAAU,EAAE,iBACb,MAAM,KACpB;IAAE,WAAW,EAAE,YAAY,CAAC,UAAU,EAAE,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAA;CAgClE,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transcriber.test.d.ts","sourceRoot":"","sources":["../../../../src/asr/transcriber.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../../../src/asr/util.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,kBAAkB,WAAY,YAAY,EAAE,8BASxD,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { type Transcript } from '@dxos/types';
|
|
2
|
+
import { type CommitFn } from './PipelineRuntime';
|
|
3
|
+
import { type StageWrite } from './Stage';
|
|
4
|
+
/**
|
|
5
|
+
* In-memory commit sink: records every {@link StageWrite} for assertions and testbench rendering.
|
|
6
|
+
*/
|
|
7
|
+
export type CaptureCommit = {
|
|
8
|
+
readonly commit: CommitFn;
|
|
9
|
+
readonly writes: StageWrite[];
|
|
10
|
+
};
|
|
11
|
+
export declare const captureCommit: () => CaptureCommit;
|
|
12
|
+
/**
|
|
13
|
+
* ECHO commit: applies a stage's output to live objects. Block patches are applied to the block
|
|
14
|
+
* objects in the window slice the stage was invoked with (the runtime passes that slice); the
|
|
15
|
+
* transcript patch is applied to the `Transcript` object. Mutating a live ECHO object propagates
|
|
16
|
+
* via reactivity; a containing `Obj.update` transaction (when present) batches the writes.
|
|
17
|
+
*/
|
|
18
|
+
export declare const makeEchoCommit: (transcript: Transcript.Transcript) => CommitFn;
|
|
19
|
+
//# sourceMappingURL=dispatch.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dispatch.d.ts","sourceRoot":"","sources":["../../../src/dispatch.ts"],"names":[],"mappings":"AAOA,OAAO,EAAqB,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAEjE,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,KAAK,UAAU,EAAE,MAAM,SAAS,CAAC;AAE1C;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC;CAC/B,CAAC;AAEF,eAAO,MAAM,aAAa,QAAO,aAOhC,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,cAAc,eACZ,UAAU,CAAC,UAAU,KAAG,QAmBjC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dispatch.test.d.ts","sourceRoot":"","sources":["../../../src/dispatch.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export * from './TranscriptEvent';
|
|
2
|
+
export * from './Stage';
|
|
3
|
+
export * from './lookup';
|
|
4
|
+
export * from './PipelineConfig';
|
|
5
|
+
export * from './model-routing';
|
|
6
|
+
export * from './PipelineRuntime';
|
|
7
|
+
export * from './live';
|
|
8
|
+
export * from './asr';
|
|
9
|
+
export * from './dispatch';
|
|
10
|
+
export * from './stages';
|
|
11
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/index.ts"],"names":[],"mappings":"AAIA,cAAc,mBAAmB,CAAC;AAClC,cAAc,SAAS,CAAC;AACxB,cAAc,UAAU,CAAC;AACzB,cAAc,kBAAkB,CAAC;AACjC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAClC,cAAc,QAAQ,CAAC;AACvB,cAAc,OAAO,CAAC;AACtB,cAAc,YAAY,CAAC;AAC3B,cAAc,UAAU,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { type ContentBlock } from '@dxos/types';
|
|
2
|
+
import { type RunOptions } from './PipelineRuntime';
|
|
3
|
+
/**
|
|
4
|
+
* Imperative handle to a running pipeline fed by a live ASR source. The source pushes events as they
|
|
5
|
+
* arrive rather than yielding a pre-materialized stream.
|
|
6
|
+
*/
|
|
7
|
+
export type LivePipeline = {
|
|
8
|
+
/** Append a transcript block produced by the ASR source. */
|
|
9
|
+
block: (block: ContentBlock.Transcript) => void;
|
|
10
|
+
/** Report that the speaker has been silent for `sinceMs` (triggers on-silence stages). */
|
|
11
|
+
silence: (sinceMs: number) => void;
|
|
12
|
+
/** Emit a periodic tick (triggers periodic stages). */
|
|
13
|
+
tick: () => void;
|
|
14
|
+
/** Close the source; resolves once the stream ends and all in-flight stages have drained. */
|
|
15
|
+
end: () => Promise<void>;
|
|
16
|
+
};
|
|
17
|
+
/**
|
|
18
|
+
* Drives {@link PipelineRuntime} from a live ASR source: blocks pushed via the returned handle are
|
|
19
|
+
* enqueued onto the runtime's source stream; `end` closes the stream so the run drains and resolves.
|
|
20
|
+
* This is the single orchestration entry point for live capture — callers supply only the stages,
|
|
21
|
+
* commit sink, and (optional) lookup, mirroring a scripted `PipelineRuntime.run`.
|
|
22
|
+
*/
|
|
23
|
+
export declare const runLivePipeline: (options: Omit<RunOptions, 'source'>) => LivePipeline;
|
|
24
|
+
//# sourceMappingURL=live.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"live.d.ts","sourceRoot":"","sources":["../../../src/live.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAAmB,KAAK,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAGrE;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,4DAA4D;IAC5D,KAAK,EAAE,CAAC,KAAK,EAAE,YAAY,CAAC,UAAU,KAAK,IAAI,CAAC;IAChD,0FAA0F;IAC1F,OAAO,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,uDAAuD;IACvD,IAAI,EAAE,MAAM,IAAI,CAAC;IACjB,6FAA6F;IAC7F,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B,CAAC;AAOF;;;;;GAKG;AACH,eAAO,MAAM,eAAe,YAAa,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,KAAG,YAerE,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"live.test.d.ts","sourceRoot":"","sources":["../../../src/live.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { type Database, Ref } from '@dxos/echo';
|
|
2
|
+
/**
|
|
3
|
+
* A candidate match for a surface noun: the matched object's id, a ref consumers can persist, and a
|
|
4
|
+
* relevance `score` (higher = better) from the search backend's match rank.
|
|
5
|
+
*/
|
|
6
|
+
export type EntityMatch = {
|
|
7
|
+
id: string;
|
|
8
|
+
ref: Ref.Ref<any>;
|
|
9
|
+
score: number;
|
|
10
|
+
};
|
|
11
|
+
/**
|
|
12
|
+
* Additional context that narrows a lookup. Grows over time (e.g. the conversation window, an
|
|
13
|
+
* expected role); backends ignore fields they do not support.
|
|
14
|
+
*/
|
|
15
|
+
export type LookupContext = {
|
|
16
|
+
/** Restrict matches to these typenames (e.g. the typename of `Person` / `Organization`). */
|
|
17
|
+
types?: string[];
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* Resolves a surface noun to its candidate matches, ranked best-first, backed by an index. The
|
|
21
|
+
* pipeline depends only on this function — not on ECHO/the database directly — so the backend
|
|
22
|
+
* (full-text, vector, a remote service, …) is an injected concern. Returning candidates (rather than
|
|
23
|
+
* a single match) lets a context-aware resolver disambiguate using the surrounding conversation.
|
|
24
|
+
* Empty when nothing matches.
|
|
25
|
+
*/
|
|
26
|
+
export type EntityLookup = (noun: string, context?: LookupContext) => Promise<EntityMatch[]>;
|
|
27
|
+
export type DatabaseLookupOptions = {
|
|
28
|
+
/** Search backend. Full-text matches proper nouns precisely; vector matches semantically. */
|
|
29
|
+
searchKind?: 'full-text' | 'vector';
|
|
30
|
+
/** Maximum candidates returned per noun. */
|
|
31
|
+
limit?: number;
|
|
32
|
+
};
|
|
33
|
+
/** An {@link EntityLookup} backed by an ECHO database query (full-text by default). */
|
|
34
|
+
export declare const makeDatabaseLookup: (db: Database.Database, { searchKind, limit }?: DatabaseLookupOptions) => EntityLookup;
|
|
35
|
+
//# sourceMappingURL=lookup.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lookup.d.ts","sourceRoot":"","sources":["../../../src/lookup.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,QAAQ,EAAsB,GAAG,EAAE,MAAM,YAAY,CAAC;AAEpE;;;GAGG;AACH,MAAM,MAAM,WAAW,GAAG;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG;IAC1B,4FAA4F;IAC5F,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,KAAK,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;AAE7F,MAAM,MAAM,qBAAqB,GAAG;IAClC,6FAA6F;IAC7F,UAAU,CAAC,EAAE,WAAW,GAAG,QAAQ,CAAC;IACpC,4CAA4C;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,uFAAuF;AACvF,eAAO,MAAM,kBAAkB,OACxB,QAAQ,CAAC,QAAQ,0BAA2C,qBAAqB,KAAQ,YAgB7F,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lookup.test.d.ts","sourceRoot":"","sources":["../../../src/lookup.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { DXN } from '@dxos/echo';
|
|
2
|
+
import { type StageConfig } from './PipelineConfig';
|
|
3
|
+
import { type Stage } from './Stage';
|
|
4
|
+
/**
|
|
5
|
+
* Resolve the model for a stage invocation. Precedence: per-stage config override → the stage's own
|
|
6
|
+
* default → the preset/pipeline default → the global {@link DEFAULT_STAGE_MODEL}.
|
|
7
|
+
*
|
|
8
|
+
* `StageConfig.model` is a validated NSID name in the schema (its format is checked, but it is not
|
|
9
|
+
* pinned to the current model catalog); it is parsed to a model DXN here for the `AiService` call.
|
|
10
|
+
*/
|
|
11
|
+
export declare const resolveModel: (stageConfig: StageConfig | undefined, stage: Stage<any, any>, presetDefault?: DXN.DXN) => DXN.DXN;
|
|
12
|
+
//# sourceMappingURL=model-routing.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-routing.d.ts","sourceRoot":"","sources":["../../../src/model-routing.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEjC,OAAO,EAAuB,KAAK,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACzE,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,SAAS,CAAC;AAErC;;;;;;GAMG;AACH,eAAO,MAAM,YAAY,gBACV,WAAW,GAAG,SAAS,SAC7B,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,kBACN,GAAG,CAAC,GAAG,KACtB,GAAG,CAAC,GAIc,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline-integration.test.d.ts","sourceRoot":"","sources":["../../../src/pipeline-integration.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import * as Effect from 'effect/Effect';
|
|
2
|
+
import { AiService } from '@dxos/ai';
|
|
3
|
+
import { DXN } from '@dxos/echo';
|
|
4
|
+
import { type ContentBlock } from '@dxos/types';
|
|
5
|
+
/**
|
|
6
|
+
* LLM-backed correction over a window of transcript blocks (the {@link makeCorrectionStage} seam).
|
|
7
|
+
* Returns an Effect requiring the {@link AiService} — the consumer provides its model layer and adapts
|
|
8
|
+
* the Effect to the stage's Promise-shaped `CorrectFn` (e.g. `EffectEx.runPromise(... .pipe(Effect.provide(layer)))`).
|
|
9
|
+
*/
|
|
10
|
+
export declare const correctWithLanguageModel: (blocks: readonly ContentBlock.Transcript[], model?: DXN.DXN) => Effect.Effect<ReadonlyArray<{
|
|
11
|
+
index: number;
|
|
12
|
+
corrected: string;
|
|
13
|
+
}>, unknown, AiService.AiService>;
|
|
14
|
+
//# sourceMappingURL=correction-llm.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"correction-llm.d.ts","sourceRoot":"","sources":["../../../../src/stages/correction-llm.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAC;AAIxC,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACjC,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAUhD;;;;GAIG;AACH,eAAO,MAAM,wBAAwB,WAC3B,SAAS,YAAY,CAAC,UAAU,EAAE,UACnC,GAAG,CAAC,GAAG,KACb,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,SAAS,CAUlB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"correction-llm.test.d.ts","sourceRoot":"","sources":["../../../../src/stages/correction-llm.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { type ContentBlock } from '@dxos/types';
|
|
2
|
+
import { type Stage } from '../Stage';
|
|
3
|
+
export type CorrectionInput = {
|
|
4
|
+
window: ContentBlock.Transcript[];
|
|
5
|
+
};
|
|
6
|
+
/**
|
|
7
|
+
* Corrects a window of transcript blocks, returning a `{ index, corrected }` patch per block it
|
|
8
|
+
* changes. The deterministic default is used unless an LLM-backed corrector is injected (see
|
|
9
|
+
* `correctWithLanguageModel`); the consumer binds the AI service and adapts it to this Promise shape.
|
|
10
|
+
*/
|
|
11
|
+
export type CorrectFn = (blocks: readonly ContentBlock.Transcript[]) => Promise<ReadonlyArray<{
|
|
12
|
+
index: number;
|
|
13
|
+
corrected: string;
|
|
14
|
+
}>>;
|
|
15
|
+
export type CorrectionOptions = {
|
|
16
|
+
/** Inject an LLM-backed corrector; omit for the deterministic per-block correction. */
|
|
17
|
+
correct?: CorrectFn;
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* Deterministic correction: trim, capitalize the first letter, ensure terminal punctuation.
|
|
21
|
+
* The LLM seam (see `run`) replaces this with cross-batch repair over the window.
|
|
22
|
+
*/
|
|
23
|
+
export declare const correctText: (text: string) => string;
|
|
24
|
+
/**
|
|
25
|
+
* Stage ①: clean up the most recent transcript block (punctuation, capitalization).
|
|
26
|
+
* Windowed for cross-batch context; emits a correction for the newest block.
|
|
27
|
+
*/
|
|
28
|
+
export declare const makeCorrectionStage: ({ correct }?: CorrectionOptions) => Stage<CorrectionInput>;
|
|
29
|
+
//# sourceMappingURL=correction.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"correction.d.ts","sourceRoot":"","sources":["../../../../src/stages/correction.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAGhD,OAAO,EAAE,KAAK,KAAK,EAAc,MAAM,UAAU,CAAC;AAElD,MAAM,MAAM,eAAe,GAAG;IAAE,MAAM,EAAE,YAAY,CAAC,UAAU,EAAE,CAAA;CAAE,CAAC;AAEpE;;;;GAIG;AACH,MAAM,MAAM,SAAS,GAAG,CACtB,MAAM,EAAE,SAAS,YAAY,CAAC,UAAU,EAAE,KACvC,OAAO,CAAC,aAAa,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAAC;AAElE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,uFAAuF;IACvF,OAAO,CAAC,EAAE,SAAS,CAAC;CACrB,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,WAAW,SAAU,MAAM,KAAG,MAY1C,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,mBAAmB,iBAAiB,iBAAiB,KAAQ,KAAK,CAAC,eAAe,CAsB7F,CAAC"}
|