@qvac/harness 0.8.0 → 0.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +97 -1
- package/README.md +49 -0
- package/dist-lib/index.d.ts +31 -2
- package/dist-lib/index.js +365 -53
- package/dist-lib/lib/audio-transcode-bare.d.ts +1 -1
- package/dist-lib/lib/audio-transcode-bare.js +22 -25
- package/dist-lib/lib/audio-transcode-node.d.ts +1 -1
- package/dist-lib/lib/audio-transcode-node.js +1 -3
- package/dist-lib/lib/capabilities/audio-resample.d.ts +1 -1
- package/dist-lib/lib/capabilities/audio-resample.js +7 -8
- package/dist-lib/lib/capabilities/completion.d.ts +5 -2
- package/dist-lib/lib/capabilities/completion.js +22 -4
- package/dist-lib/lib/capabilities/download.d.ts +3 -1
- package/dist-lib/lib/capabilities/download.js +1 -1
- package/dist-lib/lib/capabilities/preview-model-source.d.ts +1 -0
- package/dist-lib/lib/capabilities/preview-model-source.js +1 -0
- package/dist-lib/lib/capabilities/usage-breakdown.js +4 -1
- package/dist-lib/lib/capabilities/voice.d.ts +5 -1
- package/dist-lib/lib/capabilities/voice.js +94 -26
- package/dist-lib/lib/connect.d.ts +4 -1
- package/dist-lib/lib/connect.js +14 -1
- package/dist-lib/lib/contract-types.d.ts +10 -0
- package/dist-lib/lib/docx/index.d.ts +1 -1
- package/dist-lib/lib/docx/index.js +16 -24
- package/dist-lib/lib/models/download-gate.d.ts +20 -0
- package/dist-lib/lib/models/download-gate.js +73 -0
- package/dist-lib/lib/models/memory-cap.d.ts +13 -0
- package/dist-lib/lib/models/memory-cap.js +21 -0
- package/dist-lib/lib/models/model-catalog-constants.d.ts +3 -1
- package/dist-lib/lib/models/model-catalog-constants.js +9 -1
- package/dist-lib/lib/models/model-catalog.d.ts +6 -2
- package/dist-lib/lib/models/model-catalog.js +35 -3
- package/dist-lib/lib/models/model-handle.d.ts +8 -1
- package/dist-lib/lib/models/model-handle.js +20 -2
- package/dist-lib/lib/models/model-list.d.ts +8 -1
- package/dist-lib/lib/models/model-list.js +23 -0
- package/dist-lib/lib/models/model-pool.d.ts +33 -5
- package/dist-lib/lib/models/model-pool.js +97 -29
- package/dist-lib/lib/office/extract.d.ts +9 -0
- package/dist-lib/lib/office/extract.js +19 -0
- package/dist-lib/lib/office/index.d.ts +1 -1
- package/dist-lib/lib/office/index.js +74 -19
- package/dist-lib/lib/serve.js +18 -2
- package/dist-lib/lib/tools/exec/adapters/python/module-distributions.js +0 -3
- package/dist-lib/lib/tools/exec/adapters/python/python-adapter.d.ts +7 -50
- package/dist-lib/lib/tools/exec/adapters/python/python-adapter.js +38 -695
- package/dist-lib/lib/tools/exec/adapters/python/run-session.js +1 -85
- package/dist-lib/lib/tools/exec.js +9 -20
- package/dist-lib/lib/tools/generate-image.d.ts +4 -2
- package/dist-lib/lib/tools/generate-image.js +5 -2
- package/dist-lib/lib/tools/generate-music.d.ts +4 -2
- package/dist-lib/lib/tools/generate-music.js +5 -2
- package/dist-lib/lib/tools/office/ops.d.ts +13 -0
- package/dist-lib/lib/tools/office/ops.js +100 -0
- package/dist-lib/lib/tools/office.d.ts +12 -0
- package/dist-lib/lib/tools/office.js +486 -0
- package/dist-lib/lib/tools/translate.d.ts +3 -1
- package/dist-lib/lib/tools/translate.js +6 -1
- package/dist-lib/schema/harness.js +41 -3
- package/dist-lib/spec/bind.js +9 -0
- package/dist-lib/spec/calls.js +3 -0
- package/dist-lib/spec/hrpc/hrpc.json +39 -0
- package/dist-lib/spec/hrpc/index.js +39 -3
- package/dist-lib/spec/hrpc/messages.js +93 -10
- package/dist-lib/spec/hyperschema/index.js +93 -10
- package/dist-lib/spec/hyperschema/schema.json +57 -1
- package/dist-lib/spec/hyperschema/types.d.ts +19 -2
- package/package.json +13 -9
- package/wheels/manifest.json +1 -160
- package/worker/harness-sidecar.mjs +3 -1
- package/dist-lib/lib/office/pptx.d.ts +0 -1
- package/dist-lib/lib/office/pptx.js +0 -26
- package/dist-lib/lib/office/python.d.ts +0 -8
- package/dist-lib/lib/office/python.js +0 -32
- package/dist-lib/lib/office/xlsx.d.ts +0 -1
- package/dist-lib/lib/office/xlsx.js +0 -26
- package/dist-lib/lib/tools/exec/adapters/python/create-effect.d.ts +0 -7
- package/dist-lib/lib/tools/exec/adapters/python/create-effect.js +0 -318
- package/wheels/cffi-2.0.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl +0 -0
- package/wheels/cryptography-47.0.0-cp314-abi3-pyemscripten_2026_0_wasm32.whl +0 -0
- package/wheels/defusedxml-0.7.1-py2.py3-none-any.whl +0 -0
- package/wheels/et_xmlfile-2.0.0-py3-none-any.whl +0 -0
- package/wheels/fonttools-4.62.1-py3-none-any.whl +0 -0
- package/wheels/fpdf2-2.8.8-py3-none-any.whl +0 -0
- package/wheels/lxml-6.0.2-cp314-cp314-pyemscripten_2026_0_wasm32.whl +0 -0
- package/wheels/openpyxl-3.1.5-py2.py3-none-any.whl +0 -0
- package/wheels/pillow-12.2.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl +0 -0
- package/wheels/pycparser-3.0-py3-none-any.whl +0 -0
- package/wheels/pypdf-6.15.0-py3-none-any.whl +0 -0
- package/wheels/python_docx-1.2.0-py3-none-any.whl +0 -0
- package/wheels/python_pptx-1.0.2-py3-none-any.whl +0 -0
- package/wheels/six-1.17.0-py2.py3-none-any.whl +0 -0
- package/wheels/typing_extensions-4.15.0-py3-none-any.whl +0 -0
- package/wheels/xlsxwriter-3.2.9-py3-none-any.whl +0 -0
|
@@ -11,6 +11,7 @@ import { RUNAWAY_STOP_SEQUENCES } from '../helpers/control-tokens.js';
|
|
|
11
11
|
import { raceTimeout } from '../helpers/race-timeout.js';
|
|
12
12
|
import { resolveModelSrc } from './model-source.js';
|
|
13
13
|
import { errorMessage } from '../guards.js';
|
|
14
|
+
import { AbortedError } from '../helpers/race-abort.js';
|
|
14
15
|
export function createLoadTracker() {
|
|
15
16
|
const inflight = new Set();
|
|
16
17
|
let draining = null;
|
|
@@ -73,10 +74,14 @@ function entryOf(pool, key) {
|
|
|
73
74
|
export function modelOf(key) {
|
|
74
75
|
return key.split('\0')[0];
|
|
75
76
|
}
|
|
77
|
+
let uses = 0;
|
|
76
78
|
export function releaseModel(pool, key) {
|
|
77
79
|
if (pool.destroyed)
|
|
78
80
|
return;
|
|
79
81
|
const handle = entryOf(pool, key)?.value;
|
|
82
|
+
if (handle) {
|
|
83
|
+
handle.lastUsed = ++uses;
|
|
84
|
+
}
|
|
80
85
|
if (handle === undefined || !handle.loading) {
|
|
81
86
|
pool.release(key);
|
|
82
87
|
return;
|
|
@@ -86,8 +91,18 @@ export function releaseModel(pool, key) {
|
|
|
86
91
|
pool.release(key);
|
|
87
92
|
return;
|
|
88
93
|
}
|
|
89
|
-
|
|
94
|
+
const deferred = (deferredReleases.get(handle) ?? 0) + 1;
|
|
95
|
+
deferredReleases.set(handle, deferred);
|
|
96
|
+
if (deferred >= (entryOf(pool, key)?.refs ?? 0)) {
|
|
97
|
+
handle.abandon();
|
|
98
|
+
}
|
|
99
|
+
const settle = () => {
|
|
100
|
+
deferredReleases.set(handle, (deferredReleases.get(handle) ?? 1) - 1);
|
|
101
|
+
releaseSettled(pool, key);
|
|
102
|
+
};
|
|
103
|
+
void pending.then(settle, settle);
|
|
90
104
|
}
|
|
105
|
+
const deferredReleases = new WeakMap();
|
|
91
106
|
function releaseSettled(pool, key) {
|
|
92
107
|
if (!pool.destroyed) {
|
|
93
108
|
pool.release(key);
|
|
@@ -111,6 +126,21 @@ export function settledModels(pool) {
|
|
|
111
126
|
}
|
|
112
127
|
return out;
|
|
113
128
|
}
|
|
129
|
+
export function settledEntries(pool) {
|
|
130
|
+
const out = [];
|
|
131
|
+
for (const entry of pool.resources.values()) {
|
|
132
|
+
const resource = entry.value.resource;
|
|
133
|
+
if (resource !== null) {
|
|
134
|
+
out.push({
|
|
135
|
+
key: entry.id,
|
|
136
|
+
refs: entry.refs,
|
|
137
|
+
bytes: resource.bytes,
|
|
138
|
+
lastUsed: entry.value.lastUsed
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
return out;
|
|
143
|
+
}
|
|
114
144
|
export function loadingModels(pool) {
|
|
115
145
|
const out = [];
|
|
116
146
|
for (const entry of pool.resources.values()) {
|
|
@@ -120,14 +150,19 @@ export function loadingModels(pool) {
|
|
|
120
150
|
}
|
|
121
151
|
return out;
|
|
122
152
|
}
|
|
123
|
-
function reportedHandle(notify, model, load) {
|
|
153
|
+
function reportedHandle({ notify, model, addon, hold, load }) {
|
|
124
154
|
return new ModelHandle({
|
|
125
|
-
load: () => {
|
|
155
|
+
load: async (signal) => {
|
|
156
|
+
await hold?.(model, addon, signal);
|
|
126
157
|
notify({ status: 'loading', model, catalogChanged: true });
|
|
127
158
|
return load();
|
|
128
159
|
},
|
|
129
160
|
onLoaded: () => notify({ status: 'ready', model, catalogChanged: true }),
|
|
130
|
-
onFailed: (err) =>
|
|
161
|
+
onFailed: (err) => {
|
|
162
|
+
if (err instanceof AbortedError)
|
|
163
|
+
return;
|
|
164
|
+
notify({ status: 'error', model, message: errorMessage(err), catalogChanged: true });
|
|
165
|
+
}
|
|
131
166
|
});
|
|
132
167
|
}
|
|
133
168
|
export async function leaseResidentModel(pool, wanted = () => true) {
|
|
@@ -226,6 +261,10 @@ export async function awaitModelCapacity(pool, { cap, deadlineMs, tickMs = 250,
|
|
|
226
261
|
export function evictModelWithin(pool, key, options) {
|
|
227
262
|
return evictWhereWithin(pool, (candidate) => candidate === key, options);
|
|
228
263
|
}
|
|
264
|
+
export function evictIdleWithin(pool, key, { deadlineMs }) {
|
|
265
|
+
const evicting = evictIdleWhere(pool, (candidate) => candidate === key).then(() => !isResident(pool, key));
|
|
266
|
+
return raceTimeout(evicting, { ms: deadlineMs, fallback: false });
|
|
267
|
+
}
|
|
229
268
|
export function evictModel(pool, model, options) {
|
|
230
269
|
return evictWhereWithin(pool, (key) => modelOf(key) === model, options);
|
|
231
270
|
}
|
|
@@ -318,8 +357,8 @@ export function loadClockGuard(model, sdk, { now = () => Date.now(), monoMs = mo
|
|
|
318
357
|
};
|
|
319
358
|
return { race };
|
|
320
359
|
}
|
|
321
|
-
export function createModelPool(loadSdk, notify, { deviceConfig = {}, loads = createLoadTracker() } = {}) {
|
|
322
|
-
const load = async (model, opts) => {
|
|
360
|
+
export function createModelPool(loadSdk, notify, { deviceConfig = {}, loads = createLoadTracker(), beforeLoad, hold } = {}) {
|
|
361
|
+
const load = async (key, model, opts) => {
|
|
323
362
|
const sdk = await loadSdk();
|
|
324
363
|
const modelSrc = await resolveModelSrc(sdk, model);
|
|
325
364
|
const resolved = resolveLoadOptions(opts, deviceConfig);
|
|
@@ -352,54 +391,80 @@ export function createModelPool(loadSdk, notify, { deviceConfig = {}, loads = cr
|
|
|
352
391
|
});
|
|
353
392
|
const { projectionModelSrc: projection, ...loggedConfig } = modelConfig;
|
|
354
393
|
hlog('llm load config', { model, projection: Boolean(projection), ...loggedConfig });
|
|
394
|
+
const admission = await beforeLoad?.({
|
|
395
|
+
key,
|
|
396
|
+
model,
|
|
397
|
+
modelSrc,
|
|
398
|
+
modelType: 'llamacpp-completion',
|
|
399
|
+
modelConfig
|
|
400
|
+
});
|
|
355
401
|
const loadStarted = Date.now();
|
|
356
|
-
const modelId = await loadClockGuard(model, sdk)
|
|
402
|
+
const modelId = await loadClockGuard(model, sdk)
|
|
403
|
+
.race(loads.track(sdk.loadModel({
|
|
357
404
|
modelSrc,
|
|
358
405
|
modelType: 'llamacpp-completion',
|
|
359
406
|
modelConfig,
|
|
360
407
|
onProgress: (progress) => notify(progressState(model, progress))
|
|
361
|
-
})))
|
|
408
|
+
})))
|
|
409
|
+
.finally(() => admission?.release());
|
|
362
410
|
hlog('llm loaded', { model, loadMs: Date.now() - loadStarted });
|
|
363
|
-
return { model, modelId, ctxSize };
|
|
411
|
+
return { model, modelId, ctxSize, bytes: admission?.bytes };
|
|
364
412
|
};
|
|
365
413
|
return new ModelPuddle({
|
|
366
|
-
open: (key, model = key, opts = {}) => reportedHandle(notify, model, () => load(model, opts)),
|
|
414
|
+
open: (key, model = key, opts = {}) => reportedHandle({ notify, model, addon: 'llm', hold, load: () => load(key, model, opts) }),
|
|
367
415
|
gc: unloadOnGc(loadSdk, notify),
|
|
368
416
|
...NO_IDLE_GC
|
|
369
417
|
});
|
|
370
418
|
}
|
|
371
|
-
export function createEmbedPool(loadSdk, notify, { faults, batchSize = DEFAULT_EMBED_BATCH_SIZE, openclCacheDir, loads = createLoadTracker() }) {
|
|
372
|
-
const load = async (model) => {
|
|
419
|
+
export function createEmbedPool(loadSdk, notify, { faults, batchSize = DEFAULT_EMBED_BATCH_SIZE, openclCacheDir, loads = createLoadTracker(), beforeLoad, hold }) {
|
|
420
|
+
const load = async (key, model) => {
|
|
373
421
|
const sdk = await loadSdk();
|
|
374
422
|
const modelSrc = await resolveModelSrc(sdk, model, 'embeddings');
|
|
375
423
|
const openclCache = faults.device !== 'cpu' && openclCacheDir ? openclCacheDir() : undefined;
|
|
376
|
-
const
|
|
424
|
+
const modelConfig = {
|
|
425
|
+
batchSize,
|
|
426
|
+
gpuLayers: faults.gpuLayers(),
|
|
427
|
+
device: faults.device,
|
|
428
|
+
openclCacheDir: openclCache || undefined
|
|
429
|
+
};
|
|
430
|
+
const admission = await beforeLoad?.({
|
|
431
|
+
key,
|
|
432
|
+
model,
|
|
377
433
|
modelSrc,
|
|
378
434
|
modelType: 'llamacpp-embedding',
|
|
379
|
-
modelConfig
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
435
|
+
modelConfig
|
|
436
|
+
});
|
|
437
|
+
const modelId = await loadClockGuard(model, sdk)
|
|
438
|
+
.race(loads.track(sdk.loadModel({
|
|
439
|
+
modelSrc,
|
|
440
|
+
modelType: 'llamacpp-embedding',
|
|
441
|
+
modelConfig,
|
|
385
442
|
onProgress: (progress) => notify(progressState(model, progress))
|
|
386
|
-
})))
|
|
387
|
-
|
|
443
|
+
})))
|
|
444
|
+
.finally(() => admission?.release());
|
|
445
|
+
return { model, modelId, bytes: admission?.bytes };
|
|
388
446
|
};
|
|
389
447
|
return new ModelPuddle({
|
|
390
|
-
open: (key, model = key) => reportedHandle(notify, model, () => load(model)),
|
|
448
|
+
open: (key, model = key) => reportedHandle({ notify, model, addon: 'embeddings', hold, load: () => load(key, model) }),
|
|
391
449
|
gc: unloadOnGc(loadSdk, notify),
|
|
392
450
|
...NO_IDLE_GC
|
|
393
451
|
});
|
|
394
452
|
}
|
|
395
|
-
export function createVoicePool(loadSdk, notify, { modelType, addon, modelConfig, modelConfigFor, openclCacheDir, loads = createLoadTracker() }) {
|
|
453
|
+
export function createVoicePool(loadSdk, notify, { modelType, addon, modelConfig, modelConfigFor, openclCacheDir, loads = createLoadTracker(), beforeLoad, hold }) {
|
|
396
454
|
const wantsGpu = modelConfig?.['useGPU'] === true;
|
|
397
455
|
let gpuRefused = false;
|
|
398
|
-
const load = async (model) => {
|
|
456
|
+
const load = async (key, model) => {
|
|
399
457
|
const sdk = await loadSdk();
|
|
400
458
|
const modelSrc = await resolveModelSrc(sdk, model, addon);
|
|
401
459
|
const tuned = modelConfigFor?.(modelSrc);
|
|
402
460
|
const loadConfig = tuned ? { ...modelConfig, ...tuned } : modelConfig;
|
|
461
|
+
const admission = await beforeLoad?.({
|
|
462
|
+
key,
|
|
463
|
+
model,
|
|
464
|
+
modelSrc,
|
|
465
|
+
modelType,
|
|
466
|
+
modelConfig: loadConfig
|
|
467
|
+
});
|
|
403
468
|
const attempt = () => {
|
|
404
469
|
const config = gpuRefused ? { ...loadConfig, useGPU: false } : loadConfig;
|
|
405
470
|
const openclCache = gpuRefused ? undefined : openclCacheDir?.();
|
|
@@ -410,17 +475,19 @@ export function createVoicePool(loadSdk, notify, { modelType, addon, modelConfig
|
|
|
410
475
|
onProgress: (progress) => notify(progressState(model, progress))
|
|
411
476
|
}));
|
|
412
477
|
};
|
|
413
|
-
const modelId = await loadClockGuard(model, sdk)
|
|
478
|
+
const modelId = await loadClockGuard(model, sdk)
|
|
479
|
+
.race(attempt().catch((err) => {
|
|
414
480
|
if (!wantsGpu || gpuRefused || loads.draining)
|
|
415
481
|
throw err;
|
|
416
482
|
hlog('voice gpu load refused, retrying on cpu', { model, modelType, error: String(err) });
|
|
417
483
|
gpuRefused = true;
|
|
418
484
|
return attempt();
|
|
419
|
-
}))
|
|
420
|
-
|
|
485
|
+
}))
|
|
486
|
+
.finally(() => admission?.release());
|
|
487
|
+
return { model, modelId, bytes: admission?.bytes };
|
|
421
488
|
};
|
|
422
489
|
return new ModelPuddle({
|
|
423
|
-
open: (key, model = key) => reportedHandle(notify, model, () => load(model)),
|
|
490
|
+
open: (key, model = key) => reportedHandle({ notify, model, addon, hold, load: () => load(key, model) }),
|
|
424
491
|
gc: unloadOnGc(loadSdk, notify),
|
|
425
492
|
...NO_IDLE_GC
|
|
426
493
|
});
|
|
@@ -445,7 +512,8 @@ function unloadOnGc(loadSdk, notify) {
|
|
|
445
512
|
try {
|
|
446
513
|
await sdk.unloadModel({ modelId });
|
|
447
514
|
}
|
|
448
|
-
catch {
|
|
515
|
+
catch (err) {
|
|
516
|
+
hlog('unload failed', { model, err: errorMessage(err) });
|
|
449
517
|
continue;
|
|
450
518
|
}
|
|
451
519
|
notify({ status: 'unloaded', model, catalogChanged: true });
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
+
export type Extract = (bytes: Uint8Array) => string;
|
|
3
|
+
export interface ExtractJob {
|
|
4
|
+
readonly fileName: string;
|
|
5
|
+
readonly signal: AbortSignal | undefined;
|
|
6
|
+
readonly extract: Extract;
|
|
7
|
+
}
|
|
8
|
+
export declare function extractWith(bytes: Uint8Array, { fileName, signal, extract }: ExtractJob): string;
|
|
9
|
+
export declare function cellText(value: unknown): string;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { errorMessage } from '../guards.js';
|
|
2
|
+
const MAX_OFFICE_BYTES = 20 * 1024 * 1024;
|
|
3
|
+
export function extractWith(bytes, { fileName, signal, extract }) {
|
|
4
|
+
if (bytes.byteLength > MAX_OFFICE_BYTES) {
|
|
5
|
+
throw new Error(`document is too large to extract: ${bytes.byteLength} bytes, limit ${MAX_OFFICE_BYTES}`);
|
|
6
|
+
}
|
|
7
|
+
if (signal?.aborted) {
|
|
8
|
+
throw new Error(`${fileName}: extraction aborted`);
|
|
9
|
+
}
|
|
10
|
+
try {
|
|
11
|
+
return extract(bytes);
|
|
12
|
+
}
|
|
13
|
+
catch (err) {
|
|
14
|
+
throw new Error(`${fileName}: ${errorMessage(err)}`);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
export function cellText(value) {
|
|
18
|
+
return value === null || value === undefined ? '' : String(value).replaceAll('\n', ' ');
|
|
19
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
-
export {
|
|
2
|
+
export { closeDocxWorker as closeOfficeWorker } from '../docx/index.ts';
|
|
3
3
|
export interface OfficeExtractOptions {
|
|
4
4
|
readonly signal?: AbortSignal | undefined;
|
|
5
5
|
readonly mimeType?: string | undefined;
|
|
@@ -1,14 +1,81 @@
|
|
|
1
1
|
import path from 'path';
|
|
2
2
|
import { extractDocxText } from '../docx/index.js';
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
|
|
6
|
-
export { closePythonWorker as closeOfficeWorker } from '../tools/exec/adapters/python/worker/active.js';
|
|
3
|
+
import { pptx, xlsx } from './engine/index.js';
|
|
4
|
+
import { cellText, extractWith } from './extract.js';
|
|
5
|
+
export { closeDocxWorker as closeOfficeWorker } from '../docx/index.js';
|
|
7
6
|
const OFFICE_MIME_EXTENSIONS = {
|
|
8
7
|
'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',
|
|
9
8
|
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': '.xlsx',
|
|
10
9
|
'application/vnd.openxmlformats-officedocument.presentationml.presentation': '.pptx'
|
|
11
10
|
};
|
|
11
|
+
const EXTRACT_CELL_LIMIT = 1_000_000;
|
|
12
|
+
function columnIndex(letters) {
|
|
13
|
+
let index = 0;
|
|
14
|
+
for (const char of letters) {
|
|
15
|
+
index = index * 26 + char.charCodeAt(0) - 64;
|
|
16
|
+
}
|
|
17
|
+
return index;
|
|
18
|
+
}
|
|
19
|
+
function columnName(index) {
|
|
20
|
+
let name = '';
|
|
21
|
+
for (let n = index; n > 0; n = Math.floor((n - 1) / 26)) {
|
|
22
|
+
name = String.fromCharCode(65 + ((n - 1) % 26)) + name;
|
|
23
|
+
}
|
|
24
|
+
return name;
|
|
25
|
+
}
|
|
26
|
+
function trimmedRow(values) {
|
|
27
|
+
const row = [...values];
|
|
28
|
+
while (row.length && !cellText(row.at(-1)).trim()) {
|
|
29
|
+
row.pop();
|
|
30
|
+
}
|
|
31
|
+
return row;
|
|
32
|
+
}
|
|
33
|
+
function workbookText(bytes) {
|
|
34
|
+
const book = xlsx.open(bytes);
|
|
35
|
+
let output = '';
|
|
36
|
+
let budget = EXTRACT_CELL_LIMIT;
|
|
37
|
+
for (const sheet of book.sheets()) {
|
|
38
|
+
if (sheet.kind !== 'worksheet' || budget <= 0)
|
|
39
|
+
continue;
|
|
40
|
+
const { range, rows, formulas } = book.read(sheet.index, { limit: budget });
|
|
41
|
+
budget -= rows.length * (rows[0]?.length ?? 0);
|
|
42
|
+
const [, letters = 'A', digits = '1'] = /^([A-Z]+)(\d+)/.exec(range ?? 'A1') ?? [];
|
|
43
|
+
const firstColumn = columnIndex(letters);
|
|
44
|
+
const firstRow = Number(digits);
|
|
45
|
+
const lines = rows
|
|
46
|
+
.map((row, r) => trimmedRow(row.map((value, c) => value ?? formulas[`${columnName(firstColumn + c)}${firstRow + r}`] ?? null)))
|
|
47
|
+
.filter((row) => row.length)
|
|
48
|
+
.map((row) => row.map(cellText).join(' | '));
|
|
49
|
+
if (lines.length) {
|
|
50
|
+
output += `Sheet: ${sheet.name}\n${lines.join('\n')}\n`;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return output;
|
|
54
|
+
}
|
|
55
|
+
function shapeBlocks(shape) {
|
|
56
|
+
if (shape.paragraphs) {
|
|
57
|
+
const text = shape.paragraphs.map((paragraph) => paragraph.text).join('\n');
|
|
58
|
+
return text.trim() ? [text] : [];
|
|
59
|
+
}
|
|
60
|
+
return (shape.rows ?? [])
|
|
61
|
+
.map((row) => row.map(cellText))
|
|
62
|
+
.filter((cells) => cells.some((cell) => cell.trim()))
|
|
63
|
+
.map((cells) => cells.join(' | '));
|
|
64
|
+
}
|
|
65
|
+
function deckText(bytes) {
|
|
66
|
+
let output = '';
|
|
67
|
+
for (const slide of pptx.open(bytes).read()) {
|
|
68
|
+
const blocks = slide.shapes.flatMap(shapeBlocks);
|
|
69
|
+
if (blocks.length) {
|
|
70
|
+
output += `Slide ${slide.index + 1}\n${blocks.join('\n')}\n`;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return output;
|
|
74
|
+
}
|
|
75
|
+
const EXTRACTORS = new Map([
|
|
76
|
+
['.xlsx', workbookText],
|
|
77
|
+
['.pptx', deckText]
|
|
78
|
+
]);
|
|
12
79
|
export async function extractOfficeText(bytes, fileName, options = {}) {
|
|
13
80
|
const mimeType = options.mimeType?.split(';')[0]?.trim().toLowerCase();
|
|
14
81
|
const suffix = path.extname(fileName).toLowerCase();
|
|
@@ -18,21 +85,9 @@ export async function extractOfficeText(bytes, fileName, options = {}) {
|
|
|
18
85
|
if (extension === '.docx') {
|
|
19
86
|
return extractDocxText(bytes, fileName, options);
|
|
20
87
|
}
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
packages: ['openpyxl==3.1.5'],
|
|
25
|
-
source: XLSX_EXTRACT,
|
|
26
|
-
signal: options.signal
|
|
27
|
-
});
|
|
28
|
-
}
|
|
29
|
-
if (extension === '.pptx') {
|
|
30
|
-
return extractWithPython(bytes, fileName, {
|
|
31
|
-
format: 'PPTX',
|
|
32
|
-
packages: ['python-pptx==1.0.2'],
|
|
33
|
-
source: PPTX_EXTRACT,
|
|
34
|
-
signal: options.signal
|
|
35
|
-
});
|
|
88
|
+
const extract = EXTRACTORS.get(extension ?? '');
|
|
89
|
+
if (extract) {
|
|
90
|
+
return extractWith(bytes, { fileName, signal: options.signal, extract });
|
|
36
91
|
}
|
|
37
92
|
throw new Error(`Unsupported Office document: ${fileName}`);
|
|
38
93
|
}
|
package/dist-lib/lib/serve.js
CHANGED
|
@@ -23,7 +23,8 @@ const MODEL_STATUSES = new Set([
|
|
|
23
23
|
'ready',
|
|
24
24
|
'error',
|
|
25
25
|
'unloading',
|
|
26
|
-
'unloaded'
|
|
26
|
+
'unloaded',
|
|
27
|
+
'held'
|
|
27
28
|
]);
|
|
28
29
|
const DOWNLOAD_PROGRESS_INTERVAL_MS = 500;
|
|
29
30
|
const WIRE_HIGH_WATER_MARK = 64;
|
|
@@ -50,6 +51,20 @@ export function serveHarness(stream, harness) {
|
|
|
50
51
|
await harness.setModelIdleTimeout?.(ms);
|
|
51
52
|
return { ok: true };
|
|
52
53
|
},
|
|
54
|
+
setModelMemoryCap: async ({ percent }) => {
|
|
55
|
+
await harness.setModelMemoryCap?.(percent);
|
|
56
|
+
return { ok: true };
|
|
57
|
+
},
|
|
58
|
+
memoryUsage: async () => {
|
|
59
|
+
if (!harness.memoryUsage) {
|
|
60
|
+
throw new Error('harness does not support memoryUsage');
|
|
61
|
+
}
|
|
62
|
+
return harness.memoryUsage();
|
|
63
|
+
},
|
|
64
|
+
setDownloadsAllowed: async ({ allowed }) => {
|
|
65
|
+
await harness.setDownloadsAllowed?.({ allowed: allowed === true });
|
|
66
|
+
return { ok: true };
|
|
67
|
+
},
|
|
53
68
|
listManagedSkills: async () => ({
|
|
54
69
|
skills: (await harness.listManagedSkills?.()) ?? []
|
|
55
70
|
}),
|
|
@@ -282,7 +297,8 @@ function modelEvents(harness) {
|
|
|
282
297
|
total,
|
|
283
298
|
message: state.message,
|
|
284
299
|
catalogChanged,
|
|
285
|
-
loaded
|
|
300
|
+
loaded,
|
|
301
|
+
addon: state.addon
|
|
286
302
|
};
|
|
287
303
|
const now = Date.now();
|
|
288
304
|
if (state.status === 'downloading') {
|
|
@@ -20,12 +20,10 @@ export const MODULE_DISTRIBUTIONS = {
|
|
|
20
20
|
cvxpy: 'cvxpy-base',
|
|
21
21
|
dateutil: 'python-dateutil',
|
|
22
22
|
dns: 'dnspython',
|
|
23
|
-
docx: 'python-docx',
|
|
24
23
|
erfa: 'pyerfa',
|
|
25
24
|
fitz: 'PyMuPDF',
|
|
26
25
|
flint: 'python-flint',
|
|
27
26
|
flirt: 'python-flirt',
|
|
28
|
-
fpdf: 'fpdf2',
|
|
29
27
|
google: 'protobuf',
|
|
30
28
|
gridfs: 'pymongo',
|
|
31
29
|
isympy: 'sympy',
|
|
@@ -33,7 +31,6 @@ export const MODULE_DISTRIBUTIONS = {
|
|
|
33
31
|
mpl_toolkits: 'matplotlib',
|
|
34
32
|
mypyc: 'mypy',
|
|
35
33
|
nacl: 'pynacl',
|
|
36
|
-
pptx: 'python-pptx',
|
|
37
34
|
pygame: 'pygame-ce',
|
|
38
35
|
pylab: 'matplotlib',
|
|
39
36
|
pysat: 'python-sat',
|
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import type { ExecAttachment } from '../../runner.ts';
|
|
2
1
|
import type { ExecutionAdapter } from '../adapter.ts';
|
|
3
2
|
import { type ExecInput } from './attachments.ts';
|
|
4
3
|
import { type ExecError } from './output-policy.ts';
|
|
5
|
-
import { type LoadPyodideFn
|
|
4
|
+
import { type LoadPyodideFn } from './pyodide.ts';
|
|
6
5
|
export declare function isPythonLanguage(language: unknown): boolean;
|
|
7
6
|
export declare function runsThroughPyodide(language: unknown, disablePyodide: boolean): boolean;
|
|
8
7
|
export declare function unwrapPythonSource(command: string): string | null;
|
|
@@ -20,70 +19,28 @@ export interface PipFromSourceHintOptions {
|
|
|
20
19
|
export declare function pipFromSourceHint(stderr: string, source: string, { packages, sandbox }?: PipFromSourceHintOptions): string | null;
|
|
21
20
|
export declare function stringLiteralProgramHint(source: string): string | null;
|
|
22
21
|
export declare function checkCollapsedCompound(source: string): string | null;
|
|
23
|
-
export declare function mangledPptxApiHint(source: string): string | null;
|
|
24
|
-
export declare function wrongRgbColorHint(source: string): string | null;
|
|
25
|
-
export declare function fpdfSizeKeywordHint(source: string): string | null;
|
|
26
|
-
export declare function fpdfKeywordErrorHint(stderr: string): string | null;
|
|
27
|
-
export declare function normalizeFpdfSource(source: string): string;
|
|
28
|
-
export declare function fpdfTextCallHint(source: string): string | null;
|
|
29
|
-
export declare function fpdfTextErrorHint(stderr: string): string | null;
|
|
30
|
-
export declare function fpdfNonLatin1Hint(source: string): string | null;
|
|
31
|
-
export declare function fpdfUnicodeErrorHint(stderr: string): string | null;
|
|
32
|
-
export declare function fpdfHorizontalSpaceHint(stderr: string): string | null;
|
|
33
|
-
export declare function rgbColorImportHint(stderr: string): string | null;
|
|
34
22
|
export declare function checkPythonSource(command: string): string | null;
|
|
35
23
|
export declare function syntaxErrorHint(stderr: string, source: string): string | null;
|
|
36
|
-
export declare function checkPptxBuildSource(source: string, outputs: readonly string[] | undefined, packages?: readonly string[] | undefined): string | null;
|
|
37
|
-
export declare function tripleQuoteAt(source: string, index: number): string | null;
|
|
38
|
-
export declare function isEscaped(source: string, index: number): boolean;
|
|
39
|
-
export declare function checkBareLengthArgs(source: string, packages?: readonly string[] | undefined): string | null;
|
|
40
|
-
export declare function checkPdfBuildSource(source: string, outputs: readonly string[] | undefined, packages?: readonly string[] | undefined): string | null;
|
|
41
|
-
export declare function stripCommentsAndDocstrings(source: string): string;
|
|
42
|
-
export declare const PDF_READ: RegExp;
|
|
43
|
-
export declare const OFFICE_READ: RegExp;
|
|
44
|
-
export declare function checkUnstagedRead(source: string, inputs: readonly Pick<ExecInput, 'path'>[] | undefined): string | null;
|
|
45
24
|
export declare function missingFileHint(stderr: string): string | null;
|
|
46
25
|
export declare function parsePackages(value: unknown): string[] | ExecError | undefined;
|
|
47
26
|
export declare function parseInputs(value: unknown): ExecInput[] | ExecError | undefined;
|
|
48
27
|
export declare function parseOutputs(value: unknown): string[] | ExecError | undefined;
|
|
49
28
|
export declare function parseWorkdir(value: unknown): string | ExecError | undefined;
|
|
29
|
+
export interface OfficeImportContext {
|
|
30
|
+
readonly packages: readonly string[] | undefined;
|
|
31
|
+
readonly toolNames: readonly string[] | undefined;
|
|
32
|
+
}
|
|
33
|
+
export declare function officeModuleHint(stderr: string, toolNames: readonly string[] | undefined): string | null;
|
|
34
|
+
export declare function officeImportError(source: string, { packages, toolNames }: OfficeImportContext): string | null;
|
|
50
35
|
export interface MissingModuleHintOptions {
|
|
51
36
|
readonly packages?: readonly string[] | undefined;
|
|
52
37
|
readonly sandbox?: boolean;
|
|
53
38
|
readonly network?: boolean;
|
|
54
39
|
}
|
|
55
40
|
export declare function missingModuleHint(stderr: string, { packages, sandbox, network }?: MissingModuleHintOptions): string | null;
|
|
56
|
-
export declare function nonePlaceholderHint(stderr: string): string | null;
|
|
57
|
-
export declare function slideShapesHint(stderr: string): string | null;
|
|
58
|
-
export declare function shapeFontHint(stderr: string): string | null;
|
|
59
|
-
export declare function shapesBackgroundHint(stderr: string): string | null;
|
|
60
|
-
export declare function paragraphFillHint(stderr: string): string | null;
|
|
61
|
-
export declare function readOnlyPropertyHint(stderr: string): string | null;
|
|
62
|
-
export declare function paragraphIndexHint(stderr: string, outputs: readonly string[]): string | null;
|
|
63
41
|
export declare function shellNameHint(stderr: string): string | null;
|
|
64
42
|
export declare function noProcessHint(stderr: string, sandbox?: boolean): string | null;
|
|
65
43
|
export declare function noNetworkHint(stderr: string, tools?: readonly string[], network?: boolean): string | null;
|
|
66
44
|
export declare function nativeWheelHint(stderr: string): string | null;
|
|
67
45
|
export declare function packageFetchFailedHint(stderr: string): string | null;
|
|
68
|
-
export declare const IMAGE_INPUT: RegExp;
|
|
69
|
-
export declare function pptxHasEmbeddedMedia(bytes: Uint8Array): boolean;
|
|
70
|
-
export declare function softFailedDeckHint(stdout: string, exitCode: number, source: string): string | null;
|
|
71
|
-
export interface DeckImagesOptions {
|
|
72
|
-
readonly inputs: readonly {
|
|
73
|
-
path: string;
|
|
74
|
-
}[] | undefined;
|
|
75
|
-
readonly outputs: readonly string[] | undefined;
|
|
76
|
-
readonly source: string;
|
|
77
|
-
}
|
|
78
|
-
export declare function stagedImagesMissingFromDeck(result: Pick<PythonRunResult, 'exitCode' | 'files'>, { inputs, outputs, source }: DeckImagesOptions): string | null;
|
|
79
|
-
export declare function checkStagedImagesUsed(source: string, inputs: readonly Pick<ExecInput, 'path'>[] | undefined, outputs: readonly string[] | undefined): string | null;
|
|
80
|
-
export interface TurnImagesStagedOptions {
|
|
81
|
-
readonly inputs: readonly ExecInput[] | undefined;
|
|
82
|
-
readonly outputs: readonly string[] | undefined;
|
|
83
|
-
readonly turnImages: readonly {
|
|
84
|
-
path: string;
|
|
85
|
-
}[] | undefined;
|
|
86
|
-
}
|
|
87
|
-
export declare function checkTurnImagesStaged(source: string, { inputs, outputs, turnImages }: TurnImagesStagedOptions): string | null;
|
|
88
|
-
export declare function deckDeliveredStop(attachments: readonly Pick<ExecAttachment, 'fileName'>[] | undefined, exitCode: number, source?: string): string | null;
|
|
89
46
|
export declare function createPythonAdapter(disabled: boolean, loadPyodide?: LoadPyodideFn, network?: boolean): ExecutionAdapter;
|