@pipedog/core 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audio/resamplers/engine.d.ts +60 -0
- package/dist/audio/resamplers/engine.js +89 -0
- package/dist/audio/resamplers/sinc_engine.d.ts +10 -0
- package/dist/audio/resamplers/sinc_engine.js +235 -0
- package/dist/audio/resamplers/soxr_resampler.d.ts +10 -6
- package/dist/audio/resamplers/soxr_resampler.js +17 -7
- package/dist/audio/resamplers/soxr_stream_resampler.d.ts +9 -6
- package/dist/audio/resamplers/soxr_stream_resampler.js +24 -11
- package/dist/audio/turn/smart_turn/local_smart_turn_v3.js +16 -8
- package/dist/audio/vad/vad_analyzer.d.ts +11 -3
- package/dist/audio/vad/vad_analyzer.js +72 -34
- package/dist/pipeline/worker.d.ts +7 -0
- package/dist/pipeline/worker.js +40 -23
- package/dist/utils/file_storage.d.ts +37 -21
- package/dist/utils/file_storage.js +87 -9
- package/dist/utils/frame_queue.js +2 -2
- package/dist/utils/image.d.ts +0 -1
- package/dist/utils/image.js +37 -4
- package/dist/utils/jpeg_encoder.d.ts +14 -0
- package/dist/utils/jpeg_encoder.js +315 -0
- package/dist/utils/text/transforms/acronyms.d.ts +21 -0
- package/dist/utils/text/transforms/acronyms.js +31 -0
- package/dist/utils/text/transforms/currency.d.ts +16 -0
- package/dist/utils/text/transforms/currency.js +56 -0
- package/dist/utils/text/transforms/dates.d.ts +18 -0
- package/dist/utils/text/transforms/dates.js +102 -0
- package/dist/utils/text/transforms/email.d.ts +17 -0
- package/dist/utils/text/transforms/email.js +35 -0
- package/dist/utils/text/transforms/index.d.ts +37 -0
- package/dist/utils/text/transforms/index.js +42 -0
- package/dist/utils/text/transforms/numbers.d.ts +25 -0
- package/dist/utils/text/transforms/numbers.js +59 -0
- package/dist/utils/text/transforms/percentages.d.ts +16 -0
- package/dist/utils/text/transforms/percentages.js +32 -0
- package/dist/utils/text/transforms/phone.d.ts +21 -0
- package/dist/utils/text/transforms/phone.js +39 -0
- package/dist/utils/text/transforms/pronunciations.js +2 -1
- package/dist/utils/text/transforms/replacements.d.ts +39 -0
- package/dist/utils/text/transforms/replacements.js +46 -0
- package/dist/utils/text/transforms/strip_markdown.d.ts +25 -0
- package/dist/utils/text/transforms/strip_markdown.js +60 -0
- package/dist/utils/text/transforms/units.d.ts +22 -0
- package/dist/utils/text/transforms/units.js +81 -0
- package/dist/utils/text/transforms/voice_formatter.d.ts +86 -0
- package/dist/utils/text/transforms/voice_formatter.js +98 -0
- package/dist/utils/text/unicode_digits.d.ts +3 -0
- package/dist/utils/text/unicode_digits.js +34 -0
- package/dist/utils/tracing/service_decorators.js +2 -2
- package/dist/utils/tracing/setup.d.ts +18 -6
- package/dist/utils/tracing/setup.js +63 -13
- package/dist/utils/tracing/turn_trace_observer.d.ts +5 -0
- package/dist/utils/tracing/turn_trace_observer.js +18 -4
- package/dist/utils/yaml.d.ts +151 -36
- package/dist/utils/yaml.js +527 -12
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/workers/proxy/index.d.ts +3 -0
- package/dist/workers/proxy/index.js +9 -0
- package/dist/workers/proxy/websocket/client.d.ts +110 -0
- package/dist/workers/proxy/websocket/client.js +196 -0
- package/dist/workers/proxy/websocket/index.d.ts +3 -0
- package/dist/workers/proxy/websocket/index.js +10 -0
- package/package.json +18 -3
|
@@ -135,9 +135,29 @@ export class VADAnalyzer {
|
|
|
135
135
|
* @returns Current VAD state after processing the buffer.
|
|
136
136
|
*/
|
|
137
137
|
async analyzeAudio(buffer) {
|
|
138
|
-
|
|
138
|
+
// One buffer at a time, like upstream's single-thread executor: an
|
|
139
|
+
// asynchronous model must not interleave two buffers' windows.
|
|
140
|
+
while (this.#analyzing)
|
|
141
|
+
await this.#analyzing;
|
|
142
|
+
const state = this._runAnalyzer(buffer);
|
|
143
|
+
if (!isPromiseLike(state))
|
|
144
|
+
return state;
|
|
145
|
+
const done = Promise.resolve(state).then(() => { }, () => { });
|
|
146
|
+
this.#analyzing = done;
|
|
147
|
+
try {
|
|
148
|
+
return await state;
|
|
149
|
+
}
|
|
150
|
+
finally {
|
|
151
|
+
if (this.#analyzing === done)
|
|
152
|
+
this.#analyzing = null;
|
|
153
|
+
}
|
|
139
154
|
}
|
|
140
|
-
|
|
155
|
+
#analyzing = null;
|
|
156
|
+
/**
|
|
157
|
+
* Analyze audio buffer and return current VAD state.
|
|
158
|
+
*
|
|
159
|
+
* Synchronous when `voiceConfidence` is; otherwise returns a promise.
|
|
160
|
+
*/
|
|
141
161
|
_runAnalyzer(buffer) {
|
|
142
162
|
const combined = new Uint8Array(this._vadBuffer.length + buffer.length);
|
|
143
163
|
combined.set(this._vadBuffer, 0);
|
|
@@ -147,43 +167,22 @@ export class VADAnalyzer {
|
|
|
147
167
|
if (this._vadBuffer.length < numRequiredBytes) {
|
|
148
168
|
return this._vadState;
|
|
149
169
|
}
|
|
170
|
+
return this.#analyzeWindows();
|
|
171
|
+
}
|
|
172
|
+
/** Score every complete window in the buffer, then settle the state. */
|
|
173
|
+
#analyzeWindows() {
|
|
174
|
+
const numRequiredBytes = this._vadFramesNumBytes;
|
|
150
175
|
while (this._vadBuffer.length >= numRequiredBytes) {
|
|
151
176
|
const audioFrames = this._vadBuffer.slice(0, numRequiredBytes);
|
|
152
177
|
this._vadBuffer = this._vadBuffer.subarray(numRequiredBytes);
|
|
153
178
|
const confidence = this.voiceConfidence(audioFrames);
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
case VADState.QUIET:
|
|
160
|
-
this._vadState = VADState.STARTING;
|
|
161
|
-
this._vadStartingCount = 1;
|
|
162
|
-
break;
|
|
163
|
-
case VADState.STARTING:
|
|
164
|
-
this._vadStartingCount += 1;
|
|
165
|
-
break;
|
|
166
|
-
case VADState.STOPPING:
|
|
167
|
-
this._vadState = VADState.SPEAKING;
|
|
168
|
-
this._vadStoppingCount = 0;
|
|
169
|
-
break;
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
else {
|
|
173
|
-
switch (this._vadState) {
|
|
174
|
-
case VADState.STARTING:
|
|
175
|
-
this._vadState = VADState.QUIET;
|
|
176
|
-
this._vadStartingCount = 0;
|
|
177
|
-
break;
|
|
178
|
-
case VADState.SPEAKING:
|
|
179
|
-
this._vadState = VADState.STOPPING;
|
|
180
|
-
this._vadStoppingCount = 1;
|
|
181
|
-
break;
|
|
182
|
-
case VADState.STOPPING:
|
|
183
|
-
this._vadStoppingCount += 1;
|
|
184
|
-
break;
|
|
185
|
-
}
|
|
179
|
+
if (isPromiseLike(confidence)) {
|
|
180
|
+
return Promise.resolve(confidence).then((c) => {
|
|
181
|
+
this.#updateState(audioFrames, c);
|
|
182
|
+
return this.#analyzeWindows();
|
|
183
|
+
});
|
|
186
184
|
}
|
|
185
|
+
this.#updateState(audioFrames, confidence);
|
|
187
186
|
}
|
|
188
187
|
if (this._vadState === VADState.STARTING && this._vadStartingCount >= this._vadStartFrames) {
|
|
189
188
|
this._vadState = VADState.SPEAKING;
|
|
@@ -195,6 +194,42 @@ export class VADAnalyzer {
|
|
|
195
194
|
}
|
|
196
195
|
return this._vadState;
|
|
197
196
|
}
|
|
197
|
+
/** Advance the state machine with one window's confidence. */
|
|
198
|
+
#updateState(audioFrames, confidence) {
|
|
199
|
+
const volume = this._getSmoothedVolume(audioFrames);
|
|
200
|
+
this._prevVolume = volume;
|
|
201
|
+
const speaking = confidence >= this._params.confidence && volume >= this._params.minVolume;
|
|
202
|
+
if (speaking) {
|
|
203
|
+
switch (this._vadState) {
|
|
204
|
+
case VADState.QUIET:
|
|
205
|
+
this._vadState = VADState.STARTING;
|
|
206
|
+
this._vadStartingCount = 1;
|
|
207
|
+
break;
|
|
208
|
+
case VADState.STARTING:
|
|
209
|
+
this._vadStartingCount += 1;
|
|
210
|
+
break;
|
|
211
|
+
case VADState.STOPPING:
|
|
212
|
+
this._vadState = VADState.SPEAKING;
|
|
213
|
+
this._vadStoppingCount = 0;
|
|
214
|
+
break;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
else {
|
|
218
|
+
switch (this._vadState) {
|
|
219
|
+
case VADState.STARTING:
|
|
220
|
+
this._vadState = VADState.QUIET;
|
|
221
|
+
this._vadStartingCount = 0;
|
|
222
|
+
break;
|
|
223
|
+
case VADState.SPEAKING:
|
|
224
|
+
this._vadState = VADState.STOPPING;
|
|
225
|
+
this._vadStoppingCount = 1;
|
|
226
|
+
break;
|
|
227
|
+
case VADState.STOPPING:
|
|
228
|
+
this._vadStoppingCount += 1;
|
|
229
|
+
break;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
}
|
|
198
233
|
/**
|
|
199
234
|
* Release the analyzer's resources.
|
|
200
235
|
*
|
|
@@ -203,6 +238,9 @@ export class VADAnalyzer {
|
|
|
203
238
|
*/
|
|
204
239
|
async cleanup() { }
|
|
205
240
|
}
|
|
241
|
+
function isPromiseLike(x) {
|
|
242
|
+
return typeof x?.then === "function";
|
|
243
|
+
}
|
|
206
244
|
/** Python's round(): half to even. */
|
|
207
245
|
function pyRound(x) {
|
|
208
246
|
const r = Math.round(x);
|
|
@@ -332,6 +332,8 @@ export declare class PipelineWorker extends BaseWorker {
|
|
|
332
332
|
protected _clock: BaseClock;
|
|
333
333
|
protected _conversationId: string | null;
|
|
334
334
|
protected _enableTracing: boolean;
|
|
335
|
+
/** `enableTracing` as given; `_enableTracing` also needs OpenTelemetry. */
|
|
336
|
+
protected _tracingRequested: boolean;
|
|
335
337
|
protected _enableTurnTracking: boolean;
|
|
336
338
|
protected _idleTimeoutSecs: number | null;
|
|
337
339
|
protected _appResources: unknown;
|
|
@@ -396,6 +398,11 @@ export declare class PipelineWorker extends BaseWorker {
|
|
|
396
398
|
* internal processors added by the pipeline worker.
|
|
397
399
|
*/
|
|
398
400
|
get pipeline(): BasePipeline;
|
|
401
|
+
/**
|
|
402
|
+
* Create the pipeline-scoped tracing context and the latency and turn trace
|
|
403
|
+
* observers (with turn tracking enabled), and return the observers to add.
|
|
404
|
+
*/
|
|
405
|
+
protected _createTracingObservers(): BaseObserver[];
|
|
399
406
|
/** The turn tracking observer if enabled, or null. */
|
|
400
407
|
get turnTrackingObserver(): TurnTrackingObserver | null;
|
|
401
408
|
/** The turn trace observer if enabled, or null. */
|
package/dist/pipeline/worker.js
CHANGED
|
@@ -33,7 +33,7 @@ import { BaseInputTransport } from "../transports/base_input.js";
|
|
|
33
33
|
import { BaseOutputTransport } from "../transports/base_output.js";
|
|
34
34
|
import { logger } from "../utils/logger.js";
|
|
35
35
|
import { runSetupHook } from "../utils/startup.js";
|
|
36
|
-
import { isTracingAvailable } from "../utils/tracing/setup.js";
|
|
36
|
+
import { isTracingAvailable, loadOpenTelemetry, } from "../utils/tracing/setup.js";
|
|
37
37
|
import { TracingContext } from "../utils/tracing/tracing_context.js";
|
|
38
38
|
import { TurnTraceObserver } from "../utils/tracing/turn_trace_observer.js";
|
|
39
39
|
import { BaseWorker, } from "../workers/base_worker.js";
|
|
@@ -213,6 +213,8 @@ export class PipelineWorker extends BaseWorker {
|
|
|
213
213
|
_clock;
|
|
214
214
|
_conversationId;
|
|
215
215
|
_enableTracing;
|
|
216
|
+
/** `enableTracing` as given; `_enableTracing` also needs OpenTelemetry. */
|
|
217
|
+
_tracingRequested;
|
|
216
218
|
_enableTurnTracking;
|
|
217
219
|
_idleTimeoutSecs;
|
|
218
220
|
_appResources;
|
|
@@ -290,7 +292,10 @@ export class PipelineWorker extends BaseWorker {
|
|
|
290
292
|
this._startTimeoutSecs = options.startTimeoutSecs ?? START_TIMEOUT_SECS;
|
|
291
293
|
this._clock = options.clock ?? new SystemClock();
|
|
292
294
|
this._conversationId = options.conversationId ?? null;
|
|
293
|
-
this.
|
|
295
|
+
this._tracingRequested = options.enableTracing ?? false;
|
|
296
|
+
// Rechecked in _setup() (after awaiting core's asynchronous import of
|
|
297
|
+
// OpenTelemetry), so building the worker early never disables tracing.
|
|
298
|
+
this._enableTracing = this._tracingRequested && isTracingAvailable();
|
|
294
299
|
this._enableTurnTracking = options.enableTurnTracking ?? true;
|
|
295
300
|
this._idleTimeoutSecs =
|
|
296
301
|
options.idleTimeoutSecs === undefined ? IDLE_TIMEOUT_SECS : options.idleTimeoutSecs;
|
|
@@ -311,21 +316,8 @@ export class PipelineWorker extends BaseWorker {
|
|
|
311
316
|
this._turnTrackingObserver = new TurnTrackingObserver();
|
|
312
317
|
observers.push(this._turnTrackingObserver);
|
|
313
318
|
}
|
|
314
|
-
if (this._enableTracing
|
|
315
|
-
|
|
316
|
-
this._tracingContext = new TracingContext();
|
|
317
|
-
// Create latency observer for tracing
|
|
318
|
-
this._userBotLatencyObserver = new UserBotLatencyObserver();
|
|
319
|
-
observers.push(this._userBotLatencyObserver);
|
|
320
|
-
// Create turn trace observer with latency tracking
|
|
321
|
-
this._turnTraceObserver = new TurnTraceObserver({
|
|
322
|
-
turnTracker: this._turnTrackingObserver,
|
|
323
|
-
latencyTracker: this._userBotLatencyObserver,
|
|
324
|
-
conversationId: this._conversationId,
|
|
325
|
-
additionalSpanAttributes: this._additionalSpanAttributes,
|
|
326
|
-
tracingContext: this._tracingContext,
|
|
327
|
-
});
|
|
328
|
-
observers.push(this._turnTraceObserver);
|
|
319
|
+
if (this._enableTracing) {
|
|
320
|
+
observers.push(...this._createTracingObservers());
|
|
329
321
|
}
|
|
330
322
|
// RTVI support
|
|
331
323
|
const enableRtvi = options.enableRtvi ?? bridged === null;
|
|
@@ -467,6 +459,28 @@ export class PipelineWorker extends BaseWorker {
|
|
|
467
459
|
get pipeline() {
|
|
468
460
|
return this._pipeline;
|
|
469
461
|
}
|
|
462
|
+
/**
|
|
463
|
+
* Create the pipeline-scoped tracing context and the latency and turn trace
|
|
464
|
+
* observers (with turn tracking enabled), and return the observers to add.
|
|
465
|
+
*/
|
|
466
|
+
_createTracingObservers() {
|
|
467
|
+
if (!this._turnTrackingObserver) {
|
|
468
|
+
return [];
|
|
469
|
+
}
|
|
470
|
+
// Create pipeline-scoped tracing context
|
|
471
|
+
this._tracingContext = new TracingContext();
|
|
472
|
+
// Create latency observer for tracing
|
|
473
|
+
this._userBotLatencyObserver = new UserBotLatencyObserver();
|
|
474
|
+
// Create turn trace observer with latency tracking
|
|
475
|
+
this._turnTraceObserver = new TurnTraceObserver({
|
|
476
|
+
turnTracker: this._turnTrackingObserver,
|
|
477
|
+
latencyTracker: this._userBotLatencyObserver,
|
|
478
|
+
conversationId: this._conversationId,
|
|
479
|
+
additionalSpanAttributes: this._additionalSpanAttributes,
|
|
480
|
+
tracingContext: this._tracingContext,
|
|
481
|
+
});
|
|
482
|
+
return [this._userBotLatencyObserver, this._turnTraceObserver];
|
|
483
|
+
}
|
|
470
484
|
/** The turn tracking observer if enabled, or null. */
|
|
471
485
|
get turnTrackingObserver() {
|
|
472
486
|
return this._turnTrackingObserver;
|
|
@@ -1144,12 +1158,6 @@ export class PipelineWorker extends BaseWorker {
|
|
|
1144
1158
|
// the audio the EndFrame exists to play out. A processor that could
|
|
1145
1159
|
// hold it up watches for that itself.
|
|
1146
1160
|
await this._pipelineEndEvent.wait();
|
|
1147
|
-
// In asyncio the processor that pushed the EndFrame keeps running until
|
|
1148
|
-
// it next suspends, so work it does right after pushing it (e.g. the
|
|
1149
|
-
// user aggregator flushing its turn at session end) completes before
|
|
1150
|
-
// teardown. Promise continuations interleave instead, so yield one
|
|
1151
|
-
// macrotask to let that processor finish before cleaning up.
|
|
1152
|
-
await sleep(0);
|
|
1153
1161
|
logger.debug(`${this}: ${frame} reached the end of the pipeline, pipeline is closing.`);
|
|
1154
1162
|
}
|
|
1155
1163
|
this._pipelineEndEvent.clear();
|
|
@@ -1198,6 +1206,15 @@ export class PipelineWorker extends BaseWorker {
|
|
|
1198
1206
|
await super.setup(this._taskManager ?? params.taskManager);
|
|
1199
1207
|
// Do any additional pipeline worker setup externally.
|
|
1200
1208
|
await this._loadSetupFiles();
|
|
1209
|
+
// Core imports OpenTelemetry asynchronously: a worker built before that
|
|
1210
|
+
// import finished (e.g. at module top level) turns tracing on here,
|
|
1211
|
+
// before the observers start and the processors get `enableTracing`.
|
|
1212
|
+
if (this._tracingRequested && !this._enableTracing && (await loadOpenTelemetry())) {
|
|
1213
|
+
this._enableTracing = true;
|
|
1214
|
+
for (const observer of this._createTracingObservers()) {
|
|
1215
|
+
this._observer.addObserver(observer);
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1201
1218
|
// Start worker observer.
|
|
1202
1219
|
await this._observer.setup(this.taskManager);
|
|
1203
1220
|
// Setup processors
|
|
@@ -1,21 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Pluggable storage for files uploaded by clients and referenced by URL later.
|
|
3
|
-
*
|
|
4
|
-
* `FileStorage` is the shared interface between the development runner's
|
|
5
|
-
* `POST /files` upload endpoint, which saves uploads and hands the resulting
|
|
6
|
-
* URL back to the client, and the LLM service's `FileResolver`, which loads
|
|
7
|
-
* the bytes back when the client references that URL in a send-file message.
|
|
8
|
-
* Passing the same storage instance to both means they agree on where uploads
|
|
9
|
-
* live without each needing its own copy of the configuration, and lets a
|
|
10
|
-
* deployment swap in a backend other than local disk by implementing this
|
|
11
|
-
* interface.
|
|
12
|
-
*
|
|
13
|
-
* The URL is the contract: `FileStorage.save` returns one, callers pass it
|
|
14
|
-
* back unmodified, and each implementation mints whatever form it can later
|
|
15
|
-
* resolve — `pipecat:<id>` for `LocalFileStorage`, or e.g. `gs://` / `s3://`
|
|
16
|
-
* for a custom cloud backend (which the LLM provider may then even fetch
|
|
17
|
-
* itself, without `FileStorage.load` being involved at all).
|
|
18
|
-
*/
|
|
19
1
|
/**
|
|
20
2
|
* Thrown by `FileStorage.load` when a URL is invalid or no longer stored
|
|
21
3
|
* (Python's built-in `FileNotFoundError`).
|
|
@@ -60,6 +42,29 @@ export declare abstract class FileStorage {
|
|
|
60
42
|
*/
|
|
61
43
|
abstract delete(fileUrl: string): Promise<void>;
|
|
62
44
|
}
|
|
45
|
+
/**
|
|
46
|
+
* The file operations `LocalFileStorage` needs: a subset of
|
|
47
|
+
* `node:fs/promises`, which satisfies it as is
|
|
48
|
+
* (`import * as fs from "node:fs/promises"`; Deno and Bun provide it too).
|
|
49
|
+
* Core has no filesystem access of its own, so the storage takes one.
|
|
50
|
+
*/
|
|
51
|
+
export interface LocalFileSystem {
|
|
52
|
+
mkdir(path: string, options: {
|
|
53
|
+
recursive: true;
|
|
54
|
+
}): Promise<unknown>;
|
|
55
|
+
writeFile(path: string, data: Uint8Array): Promise<void>;
|
|
56
|
+
readFile(path: string): Promise<Uint8Array>;
|
|
57
|
+
unlink(path: string): Promise<void>;
|
|
58
|
+
readdir(path: string, options: {
|
|
59
|
+
withFileTypes: true;
|
|
60
|
+
}): Promise<Array<{
|
|
61
|
+
name: string;
|
|
62
|
+
isFile(): boolean;
|
|
63
|
+
}>>;
|
|
64
|
+
stat(path: string): Promise<{
|
|
65
|
+
mtimeMs: number;
|
|
66
|
+
}>;
|
|
67
|
+
}
|
|
63
68
|
export interface LocalFileStorageOptions {
|
|
64
69
|
/** Directory to store files in. Created on first save if missing. */
|
|
65
70
|
folder: string;
|
|
@@ -68,6 +73,11 @@ export interface LocalFileStorageOptions {
|
|
|
68
73
|
* once this is exceeded. Set to 0 to disable trimming.
|
|
69
74
|
*/
|
|
70
75
|
maxFiles?: number;
|
|
76
|
+
/**
|
|
77
|
+
* The filesystem to store files on, e.g. `node:fs/promises`. Required for
|
|
78
|
+
* `save`, `load` and `delete`; URL validation works without it.
|
|
79
|
+
*/
|
|
80
|
+
fs?: LocalFileSystem;
|
|
71
81
|
}
|
|
72
82
|
/**
|
|
73
83
|
* Default storage backend: files on local disk, addressed as `pipecat:<hex>`.
|
|
@@ -77,8 +87,9 @@ export interface LocalFileStorageOptions {
|
|
|
77
87
|
* the random hex suffix can't be used for path traversal and doesn't leak the
|
|
78
88
|
* original filename.
|
|
79
89
|
*
|
|
80
|
-
*
|
|
81
|
-
*
|
|
90
|
+
* Porting note: upstream writes with `aiofiles` and `pathlib`. Core cannot
|
|
91
|
+
* import `node:fs`, so the disk is reached through the `fs` option (see
|
|
92
|
+
* `LocalFileSystem`).
|
|
82
93
|
*/
|
|
83
94
|
export declare class LocalFileStorage extends FileStorage {
|
|
84
95
|
deleteAfterLoad: boolean;
|
|
@@ -86,14 +97,19 @@ export declare class LocalFileStorage extends FileStorage {
|
|
|
86
97
|
protected static readonly _URL_SUFFIX_PATTERN: RegExp;
|
|
87
98
|
protected _folder: string;
|
|
88
99
|
protected _maxFiles: number;
|
|
100
|
+
protected _fsImpl: LocalFileSystem | null;
|
|
89
101
|
constructor(options: LocalFileStorageOptions);
|
|
90
102
|
/** Write `contents` to a new randomly-named file and return its `pipecat:` URL. */
|
|
91
|
-
save(_filename: string,
|
|
103
|
+
save(_filename: string, contents: Uint8Array): Promise<string>;
|
|
92
104
|
/** Read and return the contents stored under `fileUrl`. */
|
|
93
105
|
load(fileUrl: string): Promise<Uint8Array>;
|
|
94
106
|
/** Delete the file stored under `fileUrl`, if it exists. */
|
|
95
107
|
delete(fileUrl: string): Promise<void>;
|
|
96
108
|
/** Validate `fileUrl` and return the on-disk filename it maps to. */
|
|
97
109
|
protected _resolveSuffix(fileUrl: string): string;
|
|
110
|
+
/** Keep only the most recent `maxFiles` files; delete oldest by mtime. */
|
|
111
|
+
protected _trim(): Promise<void>;
|
|
112
|
+
protected _path(name: string): string;
|
|
113
|
+
protected _fs(): LocalFileSystem;
|
|
98
114
|
}
|
|
99
115
|
//# sourceMappingURL=file_storage.d.ts.map
|
|
@@ -21,6 +21,8 @@
|
|
|
21
21
|
* for a custom cloud backend (which the LLM provider may then even fetch
|
|
22
22
|
* itself, without `FileStorage.load` being involved at all).
|
|
23
23
|
*/
|
|
24
|
+
import { rethrowIfCancelled } from "@pipedog/asyncio";
|
|
25
|
+
import { logger } from "./logger.js";
|
|
24
26
|
/**
|
|
25
27
|
* Thrown by `FileStorage.load` when a URL is invalid or no longer stored
|
|
26
28
|
* (Python's built-in `FileNotFoundError`).
|
|
@@ -41,7 +43,14 @@ export class FileStorage {
|
|
|
41
43
|
*/
|
|
42
44
|
deleteAfterLoad = false;
|
|
43
45
|
}
|
|
44
|
-
|
|
46
|
+
/** Whether `e` is a filesystem "no such file" error (`ENOENT`). */
|
|
47
|
+
function isNotFound(e) {
|
|
48
|
+
return typeof e === "object" && e !== null && e.code === "ENOENT";
|
|
49
|
+
}
|
|
50
|
+
/** `uuid.uuid4().hex`. */
|
|
51
|
+
function uuid4Hex() {
|
|
52
|
+
return crypto.randomUUID().replaceAll("-", "");
|
|
53
|
+
}
|
|
45
54
|
/**
|
|
46
55
|
* Default storage backend: files on local disk, addressed as `pipecat:<hex>`.
|
|
47
56
|
*
|
|
@@ -50,8 +59,9 @@ const NOT_PORTED = "not ported yet: LocalFileStorage needs filesystem access, wh
|
|
|
50
59
|
* the random hex suffix can't be used for path traversal and doesn't leak the
|
|
51
60
|
* original filename.
|
|
52
61
|
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
62
|
+
* Porting note: upstream writes with `aiofiles` and `pathlib`. Core cannot
|
|
63
|
+
* import `node:fs`, so the disk is reached through the `fs` option (see
|
|
64
|
+
* `LocalFileSystem`).
|
|
55
65
|
*/
|
|
56
66
|
export class LocalFileStorage extends FileStorage {
|
|
57
67
|
// Every pipecat: URL is an upload this backend owns, so the file is
|
|
@@ -61,31 +71,57 @@ export class LocalFileStorage extends FileStorage {
|
|
|
61
71
|
static _URL_SUFFIX_PATTERN = /^[0-9a-f]{32}$/;
|
|
62
72
|
_folder;
|
|
63
73
|
_maxFiles;
|
|
74
|
+
_fsImpl;
|
|
64
75
|
constructor(options) {
|
|
65
76
|
super();
|
|
66
77
|
this._folder = options.folder;
|
|
67
78
|
this._maxFiles = options.maxFiles ?? 10;
|
|
79
|
+
this._fsImpl = options.fs ?? null;
|
|
68
80
|
}
|
|
69
81
|
/** Write `contents` to a new randomly-named file and return its `pipecat:` URL. */
|
|
70
|
-
async save(_filename,
|
|
71
|
-
|
|
82
|
+
async save(_filename, contents) {
|
|
83
|
+
const fs = this._fs();
|
|
84
|
+
await fs.mkdir(this._folder, { recursive: true });
|
|
85
|
+
const suffix = uuid4Hex();
|
|
86
|
+
await fs.writeFile(this._path(suffix), contents);
|
|
87
|
+
await this._trim();
|
|
88
|
+
return `${LocalFileStorage._URL_PREFIX}${suffix}`;
|
|
72
89
|
}
|
|
73
90
|
/** Read and return the contents stored under `fileUrl`. */
|
|
74
91
|
async load(fileUrl) {
|
|
75
|
-
this._resolveSuffix(fileUrl);
|
|
76
|
-
|
|
92
|
+
const suffix = this._resolveSuffix(fileUrl);
|
|
93
|
+
try {
|
|
94
|
+
return await this._fs().readFile(this._path(suffix));
|
|
95
|
+
}
|
|
96
|
+
catch (e) {
|
|
97
|
+
rethrowIfCancelled(e);
|
|
98
|
+
if (isNotFound(e))
|
|
99
|
+
throw new FileNotFoundError(fileUrl);
|
|
100
|
+
throw e;
|
|
101
|
+
}
|
|
77
102
|
}
|
|
78
103
|
/** Delete the file stored under `fileUrl`, if it exists. */
|
|
79
104
|
async delete(fileUrl) {
|
|
105
|
+
let suffix;
|
|
80
106
|
try {
|
|
81
|
-
this._resolveSuffix(fileUrl);
|
|
107
|
+
suffix = this._resolveSuffix(fileUrl);
|
|
82
108
|
}
|
|
83
109
|
catch (e) {
|
|
84
110
|
if (e instanceof FileNotFoundError)
|
|
85
111
|
return;
|
|
86
112
|
throw e;
|
|
87
113
|
}
|
|
88
|
-
|
|
114
|
+
try {
|
|
115
|
+
// Missing is fine: a consumed upload (deleteAfterLoad) may be deleted
|
|
116
|
+
// again by session-end cleanup.
|
|
117
|
+
await this._fs().unlink(this._path(suffix));
|
|
118
|
+
}
|
|
119
|
+
catch (e) {
|
|
120
|
+
rethrowIfCancelled(e);
|
|
121
|
+
if (isNotFound(e))
|
|
122
|
+
return;
|
|
123
|
+
logger.warning(`Failed to remove uploaded file ${fileUrl}: ${e}`);
|
|
124
|
+
}
|
|
89
125
|
}
|
|
90
126
|
/** Validate `fileUrl` and return the on-disk filename it maps to. */
|
|
91
127
|
_resolveSuffix(fileUrl) {
|
|
@@ -96,5 +132,47 @@ export class LocalFileStorage extends FileStorage {
|
|
|
96
132
|
}
|
|
97
133
|
return suffix;
|
|
98
134
|
}
|
|
135
|
+
/** Keep only the most recent `maxFiles` files; delete oldest by mtime. */
|
|
136
|
+
async _trim() {
|
|
137
|
+
if (this._maxFiles <= 0) {
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
const fs = this._fs();
|
|
141
|
+
try {
|
|
142
|
+
const entries = await fs.readdir(this._folder, { withFileTypes: true });
|
|
143
|
+
const files = entries.filter((e) => e.isFile()).map((e) => e.name);
|
|
144
|
+
if (files.length <= this._maxFiles) {
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
const mtimes = new Map();
|
|
148
|
+
for (const name of files) {
|
|
149
|
+
mtimes.set(name, (await fs.stat(this._path(name))).mtimeMs);
|
|
150
|
+
}
|
|
151
|
+
const byMtime = [...files].sort((a, b) => mtimes.get(a) - mtimes.get(b));
|
|
152
|
+
for (const name of byMtime.slice(0, files.length - this._maxFiles)) {
|
|
153
|
+
try {
|
|
154
|
+
await fs.unlink(this._path(name));
|
|
155
|
+
logger.debug(`Trimmed upload ${name} from ${this._folder}`);
|
|
156
|
+
}
|
|
157
|
+
catch (e) {
|
|
158
|
+
rethrowIfCancelled(e);
|
|
159
|
+
logger.warning(`Failed to trim upload ${this._path(name)}: ${e}`);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
catch (e) {
|
|
164
|
+
rethrowIfCancelled(e);
|
|
165
|
+
logger.warning(`Failed to list uploads folder ${this._folder}: ${e}`);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
_path(name) {
|
|
169
|
+
return /[\\/]$/.test(this._folder) ? `${this._folder}${name}` : `${this._folder}/${name}`;
|
|
170
|
+
}
|
|
171
|
+
_fs() {
|
|
172
|
+
if (this._fsImpl === null) {
|
|
173
|
+
throw new Error('LocalFileStorage needs a filesystem: pass `fs` (e.g. `import * as fs from "node:fs/promises"`)');
|
|
174
|
+
}
|
|
175
|
+
return this._fsImpl;
|
|
176
|
+
}
|
|
99
177
|
}
|
|
100
178
|
//# sourceMappingURL=file_storage.js.map
|
|
@@ -23,11 +23,11 @@ export class FrameQueue extends Queue {
|
|
|
23
23
|
}
|
|
24
24
|
/** Whether any frame of the given type is in the queue. */
|
|
25
25
|
hasFrame(frameType) {
|
|
26
|
-
return this.
|
|
26
|
+
return this._queued().some((item) => this._frameGetter(item) instanceof frameType);
|
|
27
27
|
}
|
|
28
28
|
/** Whether any uninterruptible frame is currently in the queue. */
|
|
29
29
|
get hasUninterruptible() {
|
|
30
|
-
return this.
|
|
30
|
+
return this._queued().some((item) => this._isUninterruptible(item));
|
|
31
31
|
}
|
|
32
32
|
/** Remove all interruptible items, keeping uninterruptible ones. */
|
|
33
33
|
reset() {
|
package/dist/utils/image.d.ts
CHANGED
package/dist/utils/image.js
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
//
|
|
5
5
|
// Ported from pipecat: src/pipecat/utils/image.py
|
|
6
6
|
/** Image encoding helpers. */
|
|
7
|
+
import { encodeJpegRgb } from "./jpeg_encoder.js";
|
|
7
8
|
/**
|
|
8
9
|
* Encode an image for a model to look at.
|
|
9
10
|
*
|
|
@@ -22,9 +23,41 @@ export async function encodeImage(image, size, format) {
|
|
|
22
23
|
}
|
|
23
24
|
return [await encodeJpeg(image, size, format || "RGB"), "image/jpeg"];
|
|
24
25
|
}
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
26
|
+
/** Bytes per pixel of the raw pixel formats (Pillow modes) that can be encoded. */
|
|
27
|
+
const PIXEL_FORMATS = { L: 1, LA: 2, RGB: 3, RGBA: 4, RGBX: 4 };
|
|
28
|
+
/**
|
|
29
|
+
* `Image.frombytes(format, size, image).convert("RGB")`: the pixels as
|
|
30
|
+
* interleaved RGB, dropping alpha and widening grayscale.
|
|
31
|
+
*/
|
|
32
|
+
function toRgb(image, [width, height], format) {
|
|
33
|
+
const channels = PIXEL_FORMATS[format];
|
|
34
|
+
if (channels === undefined) {
|
|
35
|
+
throw new Error(`unrecognized image mode: ${format}`);
|
|
36
|
+
}
|
|
37
|
+
const pixels = width * height;
|
|
38
|
+
if (image.length < pixels * channels) {
|
|
39
|
+
throw new Error("not enough image data");
|
|
40
|
+
}
|
|
41
|
+
if (channels === 3) {
|
|
42
|
+
return image;
|
|
43
|
+
}
|
|
44
|
+
const rgb = new Uint8Array(pixels * 3);
|
|
45
|
+
for (let i = 0; i < pixels; i++) {
|
|
46
|
+
const src = i * channels;
|
|
47
|
+
if (channels <= 2) {
|
|
48
|
+
rgb[i * 3] = rgb[i * 3 + 1] = rgb[i * 3 + 2] = image[src];
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
rgb[i * 3] = image[src];
|
|
52
|
+
rgb[i * 3 + 1] = image[src + 1];
|
|
53
|
+
rgb[i * 3 + 2] = image[src + 2];
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return rgb;
|
|
57
|
+
}
|
|
58
|
+
// Upstream encodes with Pillow in a thread; there are no threads here, so the
|
|
59
|
+
// (pure TypeScript) encoder runs inline.
|
|
60
|
+
async function encodeJpeg(image, size, format) {
|
|
61
|
+
return encodeJpegRgb(toRgb(image, size, format), size[0], size[1]);
|
|
29
62
|
}
|
|
30
63
|
//# sourceMappingURL=image.js.map
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export interface EncodeJpegOptions {
|
|
2
|
+
/** Quality from 1 to 100. Defaults to 75, like Pillow. */
|
|
3
|
+
quality?: number;
|
|
4
|
+
}
|
|
5
|
+
/**
|
|
6
|
+
* Encode RGB pixels as a baseline JPEG.
|
|
7
|
+
*
|
|
8
|
+
* @param rgb Interleaved 8-bit RGB pixels, `width * height * 3` bytes, row-major.
|
|
9
|
+
* @param width Image width in pixels.
|
|
10
|
+
* @param height Image height in pixels.
|
|
11
|
+
* @returns The JPEG file bytes.
|
|
12
|
+
*/
|
|
13
|
+
export declare function encodeJpegRgb(rgb: Uint8Array, width: number, height: number, options?: EncodeJpegOptions): Uint8Array;
|
|
14
|
+
//# sourceMappingURL=jpeg_encoder.d.ts.map
|