@hraness/sponge 0.4.1 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -16
- package/dist/src/acquisition/capture.d.ts +1 -4
- package/dist/src/acquisition/capture.js +12 -23
- package/dist/src/acquisition/clip-helper.js +14 -0
- package/dist/src/acquisition/errors.d.ts +2 -1
- package/dist/src/acquisition/errors.js +4 -33
- package/dist/src/acquisition/lock.d.ts +1 -1
- package/dist/src/acquisition/lock.js +1 -1
- package/dist/src/acquisition/pins.d.ts +1 -1
- package/dist/src/acquisition/pins.js +7 -13
- package/dist/src/acquisition/process.d.ts +1 -1
- package/dist/src/acquisition/process.js +1 -2
- package/dist/src/acquisition/runtime.d.ts +0 -4
- package/dist/src/acquisition/runtime.js +12 -18
- package/dist/src/app/source.d.ts +2 -0
- package/dist/src/app/source.js +18 -0
- package/dist/src/cli/commands.js +28 -18
- package/dist/src/cli/config.d.ts +2 -1
- package/dist/src/cli/config.js +8 -8
- package/dist/src/cli/runtimes.js +1 -1
- package/dist/src/cli/worker.js +13 -10
- package/dist/src/errors.d.ts +1 -0
- package/dist/src/errors.js +2 -0
- package/dist/src/media/capture.d.ts +6 -2
- package/dist/src/media/capture.js +33 -42
- package/dist/src/models/catalog.d.ts +4 -2
- package/dist/src/models/catalog.js +39 -20
- package/dist/src/models/runtime.d.ts +11 -3
- package/dist/src/models/runtime.js +18 -11
- package/dist/src/pdf/capture-helper.js +6 -5
- package/dist/src/pdf/capture.d.ts +0 -4
- package/dist/src/pdf/capture.js +2 -14
- package/dist/src/pdf/errors.js +1 -3
- package/dist/src/worker/basic.d.ts +1 -1
- package/dist/src/worker/basic.js +5 -3
- package/native/runtime-manifest.json +14 -6
- package/package.json +1 -1
- package/dist/src/media/capture-helper.js +0 -21
- /package/dist/src/{media/capture-helper.d.ts → acquisition/clip-helper.d.ts} +0 -0
|
@@ -12,7 +12,7 @@ const installDeadlineMs = 30 * 60 * 1000;
|
|
|
12
12
|
function checkPins(pins) {
|
|
13
13
|
if (!/^[a-z0-9][a-z0-9.-]{0,99}$/.test(pins.id) || hash(pins.lockfile) !== pins.lockSha256)
|
|
14
14
|
throw new SpongeError('invalid-input');
|
|
15
|
-
for (const pin of [pins.bun, pins.
|
|
15
|
+
for (const pin of [pins.bun, pins.wordcell]) {
|
|
16
16
|
let url;
|
|
17
17
|
try {
|
|
18
18
|
url = new URL(pin.url);
|
|
@@ -36,7 +36,7 @@ function checkPins(pins) {
|
|
|
36
36
|
}
|
|
37
37
|
export function captureSetupPlan(pins) {
|
|
38
38
|
checkPins(pins);
|
|
39
|
-
const artifacts = [pins.bun, pins.
|
|
39
|
+
const artifacts = [pins.bun, pins.wordcell], downloadBytes = artifacts.reduce((n, a) => n + a.bytes, 0);
|
|
40
40
|
return {
|
|
41
41
|
artifacts,
|
|
42
42
|
downloadBytes,
|
|
@@ -46,7 +46,7 @@ export function captureSetupPlan(pins) {
|
|
|
46
46
|
installationDeadlineMs: installDeadlineMs,
|
|
47
47
|
dependencyDownloads: 'frozen-lock',
|
|
48
48
|
requiredFreeBytes: 2 * 1024 ** 3 + budget + stagingBudget + downloadBytes,
|
|
49
|
-
capabilities: ['article-http', 'selected-
|
|
49
|
+
capabilities: ['article-http', 'selected-attached-browser', 'verified-offline-images'],
|
|
50
50
|
};
|
|
51
51
|
}
|
|
52
52
|
async function download(pin, path, signal) {
|
|
@@ -116,7 +116,7 @@ export const runtimeVerification = { fullWalks: 0 };
|
|
|
116
116
|
async function inspectRuntime(root, pins) {
|
|
117
117
|
checkPins(pins);
|
|
118
118
|
await privateRoot(root);
|
|
119
|
-
const key = JSON.stringify([root, pins.id, pins.lockSha256, pins.bun.archiveExecutable, pins.bun.executableSha256]), bun = join(root, pins.bun.archiveExecutable)
|
|
119
|
+
const key = JSON.stringify([root, pins.id, pins.lockSha256, pins.bun.archiveExecutable, pins.bun.executableSha256]), bun = join(root, pins.bun.archiveExecutable);
|
|
120
120
|
// Stat receipt of the root, its receipt file and every inventoried entry. Each stat precedes any read of its entry, so a later change always shows.
|
|
121
121
|
const head = [
|
|
122
122
|
statLine('.', await lstat(root, { bigint: true })),
|
|
@@ -124,7 +124,7 @@ async function inspectRuntime(root, pins) {
|
|
|
124
124
|
];
|
|
125
125
|
const cached = verified.get(key);
|
|
126
126
|
if (cached !== undefined && cached === hash([...head, ...(await inventory(root, false)).stats].join('\n')))
|
|
127
|
-
return { root, bun,
|
|
127
|
+
return { root, bun, pins };
|
|
128
128
|
verified.delete(key);
|
|
129
129
|
runtimeVerification.fullWalks++;
|
|
130
130
|
const receipt = JSON.parse((await readConfined(root, 'runtime-receipt.json', 4096)).toString()), walked = await inventory(root);
|
|
@@ -136,7 +136,7 @@ async function inspectRuntime(root, pins) {
|
|
|
136
136
|
if (hash(await readConfined(root, pins.bun.archiveExecutable, 128 * 1024 ** 2)) !== pins.bun.executableSha256)
|
|
137
137
|
throw new SpongeError('integrity');
|
|
138
138
|
verified.set(key, hash([...head, ...walked.stats].join('\n')));
|
|
139
|
-
return { root, bun,
|
|
139
|
+
return { root, bun, pins };
|
|
140
140
|
}
|
|
141
141
|
async function provisionRuntime(options) {
|
|
142
142
|
const { pins, signal } = options, plan = captureSetupPlan(pins);
|
|
@@ -188,7 +188,7 @@ async function provisionRuntime(options) {
|
|
|
188
188
|
await download(pins.bun, join(stage, 'bun.zip'), controller.signal);
|
|
189
189
|
const unpack = await runOwned('/usr/bin/unzip', ['-q', join(stage, 'bun.zip'), '-d', stage], {
|
|
190
190
|
cwd: stage,
|
|
191
|
-
env: isolatedEnvironment(home, home
|
|
191
|
+
env: isolatedEnvironment(home, home),
|
|
192
192
|
signal: controller.signal,
|
|
193
193
|
deadlineMs: 30000,
|
|
194
194
|
maxOutputBytes: 65536,
|
|
@@ -200,16 +200,15 @@ async function provisionRuntime(options) {
|
|
|
200
200
|
if (hash(await readConfined(stage, pins.bun.archiveExecutable, 128 * 1024 ** 2)) !== pins.bun.executableSha256)
|
|
201
201
|
throw new SpongeError('integrity');
|
|
202
202
|
await chmod(bun, 0o700);
|
|
203
|
-
await download(pins.ghostget, join(stage, `hraness-ghostget-${pins.ghostget.version}.tgz`), controller.signal);
|
|
204
203
|
await download(pins.wordcell, join(stage, `hraness-wordcell-${pins.wordcell.version}.tgz`), controller.signal);
|
|
205
204
|
await writeFile(join(stage, 'package.json'), JSON.stringify({
|
|
206
205
|
private: true,
|
|
207
|
-
dependencies: { '@hraness/
|
|
206
|
+
dependencies: { '@hraness/wordcell': `./hraness-wordcell-${pins.wordcell.version}.tgz` },
|
|
208
207
|
}), { mode: 0o600 });
|
|
209
208
|
await writeFile(join(stage, 'bun.lock'), pins.lockfile, { mode: 0o600 });
|
|
210
209
|
const install = await runOwned(bun, ['install', '--frozen-lockfile', '--ignore-scripts'], {
|
|
211
210
|
cwd: stage,
|
|
212
|
-
env: isolatedEnvironment(home, home
|
|
211
|
+
env: isolatedEnvironment(home, home),
|
|
213
212
|
signal: controller.signal,
|
|
214
213
|
deadlineMs: installDeadlineMs,
|
|
215
214
|
maxOutputBytes: 65536,
|
|
@@ -219,14 +218,9 @@ async function provisionRuntime(options) {
|
|
|
219
218
|
throw new SpongeError(limitExceeded ? 'limit-exceeded' : (install.stopped ?? 'unavailable'));
|
|
220
219
|
await rm(home, { recursive: true, force: true });
|
|
221
220
|
await rm(join(stage, 'bun.zip'));
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
]) {
|
|
226
|
-
const pkg = JSON.parse((await readConfined(stage, `node_modules/@hraness/${name}/package.json`, 65536)).toString());
|
|
227
|
-
if (pkg.version !== pin.version)
|
|
228
|
-
throw new SpongeError('integrity');
|
|
229
|
-
}
|
|
221
|
+
const pkg = JSON.parse((await readConfined(stage, 'node_modules/@hraness/wordcell/package.json', 65536)).toString());
|
|
222
|
+
if (pkg.version !== pins.wordcell.version)
|
|
223
|
+
throw new SpongeError('integrity');
|
|
230
224
|
await writeFile(join(stage, 'runtime-receipt.json'), JSON.stringify({
|
|
231
225
|
schemaVersion: 1,
|
|
232
226
|
pinId: pins.id,
|
package/dist/src/app/source.d.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
/** The identity form of an http(s) locator; source keys and shareable URIs both use it. Its output pins every existing source key. */
|
|
2
2
|
export declare function normalizeWebLocator(locator: string): string;
|
|
3
|
+
/** The kind `add` assumes without --kind: PDFs by extension or arXiv path, media by known host or file extension, else an article. */
|
|
4
|
+
export declare function inferKind(input: string): 'article' | 'pdf' | 'media';
|
|
3
5
|
export declare function articleSource(locator: string, realm?: string): {
|
|
4
6
|
sourceKey: string;
|
|
5
7
|
locator: string;
|
package/dist/src/app/source.js
CHANGED
|
@@ -31,6 +31,24 @@ export function normalizeWebLocator(locator) {
|
|
|
31
31
|
// URL normalizes host casing/default ports. Preserve query order, unknown fields and fragments.
|
|
32
32
|
return url.href;
|
|
33
33
|
}
|
|
34
|
+
const mediaHosts = /^(?:(?:www|m|music)\.)?(?:youtube\.com|youtu\.be|vimeo\.com|soundcloud\.com)$/i;
|
|
35
|
+
const mediaExtensions = /\.(?:mp3|m4a|aac|wav|flac|ogg|opus|mp4|m4v|mov|webm)$/i;
|
|
36
|
+
/** The kind `add` assumes without --kind: PDFs by extension or arXiv path, media by known host or file extension, else an article. */
|
|
37
|
+
export function inferKind(input) {
|
|
38
|
+
if (/\.pdf(?:[?#]|$)/i.test(input) || input.startsWith('file:'))
|
|
39
|
+
return 'pdf';
|
|
40
|
+
try {
|
|
41
|
+
const url = new URL(input);
|
|
42
|
+
if (/^(?:www\.|export\.)?arxiv\.org$/i.test(url.hostname) && url.pathname.startsWith('/pdf/'))
|
|
43
|
+
return 'pdf';
|
|
44
|
+
if (mediaHosts.test(url.hostname) || mediaExtensions.test(url.pathname))
|
|
45
|
+
return 'media';
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
// An invalid locator is refused by its source constructor.
|
|
49
|
+
}
|
|
50
|
+
return 'article';
|
|
51
|
+
}
|
|
34
52
|
export function articleSource(locator, realm = 'public') {
|
|
35
53
|
return webSource(locator, realm, 'article');
|
|
36
54
|
}
|
package/dist/src/cli/commands.js
CHANGED
|
@@ -2,16 +2,17 @@ import { join, isAbsolute } from 'node:path';
|
|
|
2
2
|
import { randomUUID } from 'node:crypto';
|
|
3
3
|
import { readConnections } from './config.js';
|
|
4
4
|
import { pdfSource } from '../pdf/index.js';
|
|
5
|
-
import { articleSource, mediaSource } from '../app/source.js';
|
|
5
|
+
import { articleSource, inferKind, mediaSource } from '../app/source.js';
|
|
6
6
|
import { defaultCapturePins, inspectCaptureRuntime, provisionCapture, captureSetupPlan } from '../acquisition/index.js';
|
|
7
7
|
import { queueAction } from './queue.js';
|
|
8
8
|
import { portabilityAction } from './portability.js';
|
|
9
9
|
import { restoreLibrary } from '../portability/backup.js';
|
|
10
10
|
import { workerAction, serveWorker, hasDueAcquisition, workerDoctor } from './worker.js';
|
|
11
|
-
import { nativeSetupPlan, provisionNative, inspectNative } from '../models/runtime.js';
|
|
11
|
+
import { nativeSetupPlan, provisionNative, inspectNative, optionalNative } from '../models/runtime.js';
|
|
12
12
|
import { digestModel, digestSetupPlan, inspectDigest, provisionDigest } from '../models/digest.js';
|
|
13
13
|
import { withDigestServer } from '../digest/server.js';
|
|
14
14
|
import { nativeDownloadAuthGuidance } from '../models/download.js';
|
|
15
|
+
import { nativePlatformSupported } from '../models/catalog.js';
|
|
15
16
|
import { drainIndexQueue } from '../indexing/runner.js';
|
|
16
17
|
import { indexAction, overviewText } from './indexing.js';
|
|
17
18
|
import { pruneText } from '../indexing/prune-run.js';
|
|
@@ -55,6 +56,8 @@ const resolveConnection = (connections) => (realm) => {
|
|
|
55
56
|
const connection = connections.find((c) => c.realm === realm);
|
|
56
57
|
if (!connection)
|
|
57
58
|
throw new SpongeError('repair-auth');
|
|
59
|
+
if (!connection.capture)
|
|
60
|
+
throw new SpongeError('connection-retired');
|
|
58
61
|
return connection.capture;
|
|
59
62
|
};
|
|
60
63
|
/** add/refresh with --wait and run --drain: drain acquisition (one job when given), then queued indexes for run. */
|
|
@@ -67,16 +70,9 @@ async function drain(c, jobId, connections) {
|
|
|
67
70
|
throw jobFailure(store, jobId);
|
|
68
71
|
}
|
|
69
72
|
const runtime = await captureRuntime(c);
|
|
70
|
-
let asrRuntime;
|
|
71
|
-
try {
|
|
72
|
-
asrRuntime = await inspectNative('asr', c.nativeRoot, c.version);
|
|
73
|
-
}
|
|
74
|
-
catch (error) {
|
|
75
|
-
if (!['runtime-missing', 'unsupported'].includes(publicCode(error, 'integrity')))
|
|
76
|
-
throw error;
|
|
77
|
-
}
|
|
78
73
|
const results = await drainLibrary(store, {
|
|
79
|
-
asrRuntime,
|
|
74
|
+
asrRuntime: await optionalNative('asr', c.nativeRoot, c.version),
|
|
75
|
+
mediaRuntime: await optionalNative('media', c.nativeRoot, c.version),
|
|
80
76
|
language: args.flags.language,
|
|
81
77
|
forceLocal: args.flags['force-local'] === true,
|
|
82
78
|
runtime,
|
|
@@ -154,8 +150,8 @@ export const commands = [
|
|
|
154
150
|
c.emit({ digest: 'ready', model: pin.alias, auto: settings.auto, gc }, `Digest model ready: ${pin.alias}. New captures are digested in the background; turn that off with \`sponge digest --auto off\`.`);
|
|
155
151
|
return 0;
|
|
156
152
|
}
|
|
157
|
-
|
|
158
|
-
const
|
|
153
|
+
const installNative = async (kind) => {
|
|
154
|
+
const plan = await nativeSetupPlan(kind, c.version);
|
|
159
155
|
process.stderr.write(JSON.stringify({ setup: kind, ...plan }) + '\n');
|
|
160
156
|
try {
|
|
161
157
|
await provisionNative(kind, c.nativeRoot, c.version, c.signal);
|
|
@@ -165,10 +161,20 @@ export const commands = [
|
|
|
165
161
|
process.stderr.write(nativeDownloadAuthGuidance + '\n');
|
|
166
162
|
throw error;
|
|
167
163
|
}
|
|
164
|
+
};
|
|
165
|
+
if (!c.args.flags.capture) {
|
|
166
|
+
const kind = c.args.flags.asr ? 'asr' : 'embedding';
|
|
167
|
+
await installNative(kind);
|
|
168
168
|
const gc = await setupRuntimeGc(c.store, c.version);
|
|
169
169
|
c.emit({ [kind]: 'ready', gc });
|
|
170
170
|
return 0;
|
|
171
171
|
}
|
|
172
|
+
// Media goes first so a running worker never drains media jobs against a half-finished setup.
|
|
173
|
+
let media = nativePlatformSupported() ? 'ready' : 'unsupported';
|
|
174
|
+
if (media === 'ready')
|
|
175
|
+
await installNative('media').catch((error) => {
|
|
176
|
+
media = codeOf(error);
|
|
177
|
+
});
|
|
172
178
|
const plan = captureSetupPlan(defaultCapturePins);
|
|
173
179
|
process.stderr.write(`Capture setup: ${plan.downloadBytes} pinned artifact bytes, plus frozen dependency downloads. Temporary installation limit: ${plan.stagingBudgetBytes} bytes; retained runtime limit: ${plan.installationBudgetBytes} bytes; dependency installation deadline: ${plan.installationDeadlineMs / 60000} minutes. ${plan.requiredFreeBytes} free bytes required.\n`);
|
|
174
180
|
await provisionCapture({
|
|
@@ -176,8 +182,13 @@ export const commands = [
|
|
|
176
182
|
pins: defaultCapturePins,
|
|
177
183
|
signal: c.signal,
|
|
178
184
|
});
|
|
179
|
-
|
|
180
|
-
|
|
185
|
+
// Articles and PDFs run anywhere; the media helper is a native release asset and is best-effort.
|
|
186
|
+
const gc = await setupRuntimeGc(c.store, c.version), ready = media === 'ready'
|
|
187
|
+
? 'Capture ready'
|
|
188
|
+
: media === 'unsupported'
|
|
189
|
+
? 'Capture ready (media capture needs macOS 26 on Apple Silicon)'
|
|
190
|
+
: `Capture ready; media helper not installed (${media}). Articles and PDFs work; rerun \`sponge setup --capture\` for media.`;
|
|
191
|
+
c.emit({ capture: 'ready', media, gc }, 'skipped' in gc ? `${ready}\nRuntime GC skipped: ${gc.skipped}` : `${ready}\n${runtimeGcText(gc)}`);
|
|
181
192
|
return 0;
|
|
182
193
|
},
|
|
183
194
|
},
|
|
@@ -193,8 +204,7 @@ export const commands = [
|
|
|
193
204
|
const connection = args.flags.connection ? connections.find((x) => x.id === args.flags.connection) : undefined;
|
|
194
205
|
if (args.flags.connection && !connection)
|
|
195
206
|
throw new SpongeError('invalid-input');
|
|
196
|
-
const input = args.positional[0], kind = (args.flags.kind ??
|
|
197
|
-
(/\.pdf(?:[?#]|$)/i.test(input) || input.startsWith('file:') ? 'pdf' : 'article'));
|
|
207
|
+
const input = args.positional[0], kind = (args.flags.kind ?? inferKind(input));
|
|
198
208
|
const source = (kind === 'pdf' ? pdfSource : kind === 'media' ? mediaSource : articleSource)(input, connection?.realm ?? 'public');
|
|
199
209
|
const submission = store.queue.submit({ kind, ...source, idempotencyKey: 'save:' + source.sourceKey }), { jobId, itemId } = submission;
|
|
200
210
|
const job = store.queue.list().find((j) => j.id === jobId);
|
|
@@ -436,7 +446,7 @@ export const commands = [
|
|
|
436
446
|
capture = publicCode(error, 'runtime-missing');
|
|
437
447
|
}
|
|
438
448
|
const states = {};
|
|
439
|
-
for (const kind of ['asr', 'embedding']) {
|
|
449
|
+
for (const kind of ['asr', 'embedding', 'media']) {
|
|
440
450
|
try {
|
|
441
451
|
await inspectNative(kind, c.nativeRoot, c.version);
|
|
442
452
|
states[kind] = 'ready';
|
package/dist/src/cli/config.d.ts
CHANGED
|
@@ -2,7 +2,8 @@ import type { CaptureConnection } from '../acquisition/index.js';
|
|
|
2
2
|
export interface Connection {
|
|
3
3
|
id: string;
|
|
4
4
|
realm: string;
|
|
5
|
-
|
|
5
|
+
/** Absent for a retired GhostGet account connection: it still names its items' realm, but cannot capture. */
|
|
6
|
+
capture?: CaptureConnection;
|
|
6
7
|
}
|
|
7
8
|
/** Local references only. Tokens, cookies and passwords never belong in this file. */
|
|
8
9
|
export declare function readConnections(root: string): Promise<Connection[]>;
|
package/dist/src/cli/config.js
CHANGED
|
@@ -32,17 +32,17 @@ export async function readConnections(root) {
|
|
|
32
32
|
if (!entry || typeof entry.id !== 'string' || !/^[a-zA-Z0-9_-]{1,64}$/.test(entry.id) || ids.has(entry.id))
|
|
33
33
|
throw new SpongeError('invalid-input');
|
|
34
34
|
ids.add(entry.id);
|
|
35
|
-
let capture;
|
|
36
35
|
if (entry.kind === 'account' &&
|
|
37
36
|
typeof entry.stateHome === 'string' &&
|
|
38
37
|
entry.stateHome.startsWith('/') &&
|
|
39
|
-
typeof entry.accountId === 'string')
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
38
|
+
typeof entry.accountId === 'string') {
|
|
39
|
+
// The realm keeps its original derivation, so items captured through it keep their access class.
|
|
40
|
+
const legacy = { kind: 'account', stateHome: entry.stateHome, accountId: entry.accountId };
|
|
41
|
+
return { id: entry.id, realm: 'connection:' + sha256(JSON.stringify([entry.id, legacy])) };
|
|
42
|
+
}
|
|
43
|
+
if (entry.kind !== 'cdp' || typeof entry.endpoint !== 'string')
|
|
44
44
|
throw new SpongeError('invalid-input');
|
|
45
|
-
const
|
|
46
|
-
return { id: entry.id, realm, capture };
|
|
45
|
+
const capture = { kind: 'cdp', endpoint: entry.endpoint };
|
|
46
|
+
return { id: entry.id, realm: 'connection:' + sha256(JSON.stringify([entry.id, capture])), capture };
|
|
47
47
|
});
|
|
48
48
|
}
|
package/dist/src/cli/runtimes.js
CHANGED
|
@@ -11,7 +11,7 @@ async function installedReferences(version) {
|
|
|
11
11
|
const catalog = await loadNativeCatalog(version);
|
|
12
12
|
models = {
|
|
13
13
|
fingerprints: [
|
|
14
|
-
...['asr', 'embedding'].map((kind) => modelSetupPlan(nativeModelPackage(catalog, kind)).fingerprint),
|
|
14
|
+
...['asr', 'embedding', 'media'].map((kind) => modelSetupPlan(nativeModelPackage(catalog, kind)).fingerprint),
|
|
15
15
|
...digestFingerprints(),
|
|
16
16
|
],
|
|
17
17
|
};
|
package/dist/src/cli/worker.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { join } from 'node:path';
|
|
2
|
-
import { inspectNative } from '../models/runtime.js';
|
|
2
|
+
import { inspectNative, optionalNative } from '../models/runtime.js';
|
|
3
3
|
import { drainIndexQueue } from '../indexing/runner.js';
|
|
4
4
|
import { drainDigestQueue } from '../digest/runner.js';
|
|
5
5
|
import { drainLibrary } from '../worker/basic.js';
|
|
@@ -63,15 +63,18 @@ export async function drainServiceQueues(store, options, version, signal) {
|
|
|
63
63
|
const runtime = options.runtime ?? (await options.resolveCaptureRuntime?.());
|
|
64
64
|
if (!runtime)
|
|
65
65
|
throw new SpongeError('runtime-missing');
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
66
|
+
const models = join(store.root, 'runtime', 'models');
|
|
67
|
+
// Media jobs report a missing helper themselves; articles and PDFs still drain.
|
|
68
|
+
const asrRuntime = options.asrRuntime ?? (await optionalNative('asr', models, version));
|
|
69
|
+
const mediaRuntime = options.mediaRuntime ?? (await optionalNative('media', models, version));
|
|
70
|
+
results.push(...(await drainLibrary(store, {
|
|
71
|
+
...options,
|
|
72
|
+
runtime,
|
|
73
|
+
asrRuntime,
|
|
74
|
+
mediaRuntime,
|
|
75
|
+
signal,
|
|
76
|
+
allowPrivateNetwork: false,
|
|
77
|
+
})));
|
|
75
78
|
}
|
|
76
79
|
catch (error) {
|
|
77
80
|
acquisitionError = error;
|
package/dist/src/errors.d.ts
CHANGED
|
@@ -22,6 +22,7 @@ declare const parents: {
|
|
|
22
22
|
readonly 'source-too-thin': "unavailable";
|
|
23
23
|
readonly 'parent-missing': "invalid-input";
|
|
24
24
|
readonly 'target-exists': "invalid-input";
|
|
25
|
+
readonly 'connection-retired': "unsupported";
|
|
25
26
|
};
|
|
26
27
|
export type DetailCode = keyof typeof parents;
|
|
27
28
|
export type SpongeCode = ErrorCode | DetailCode;
|
package/dist/src/errors.js
CHANGED
|
@@ -36,6 +36,7 @@ const parents = {
|
|
|
36
36
|
'source-too-thin': 'unavailable',
|
|
37
37
|
'parent-missing': 'invalid-input',
|
|
38
38
|
'target-exists': 'invalid-input',
|
|
39
|
+
'connection-retired': 'unsupported',
|
|
39
40
|
};
|
|
40
41
|
export const indexCodes = [
|
|
41
42
|
'invalid-index',
|
|
@@ -86,6 +87,7 @@ export const detailHints = {
|
|
|
86
87
|
'draft-rejected': 'The model draft failed the digest checks listed above, even after one repair. Drafting is deterministic, so the same capture and model fail again; refresh the item or use another model.',
|
|
87
88
|
'model-unavailable': 'The model server did not answer or returned an error. Check that it is still running and has free GPU memory (another loaded model can exhaust it), then retry.',
|
|
88
89
|
'source-too-thin': 'The capture holds too little text to digest: under 20 words of prose, or only an archive’s metadata. Refresh the item when a fuller capture is possible.',
|
|
90
|
+
'connection-retired': 'Saved GhostGet account connections were retired in Sponge 0.5.0. Attach a signed-in browser over CDP and add it to connections.json instead.',
|
|
89
91
|
'model-busy': 'Another llama-server is running, and two loaded models can exhaust GPU memory. Wait for it to exit, then retry.',
|
|
90
92
|
};
|
|
91
93
|
/** The one error type. `message` is the code, so bare `String(error)` never carries detail, paths or causes. */
|
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
import type { Asset, ErrorEnvelope } from '../../contracts/index.js';
|
|
2
|
-
import { type CaptureRuntime } from '../acquisition/runtime.js';
|
|
3
2
|
import { runOwned } from '../acquisition/process.js';
|
|
4
3
|
import type { CaptureConnection } from '../acquisition/index.js';
|
|
5
4
|
import type { AudioInput } from '../inference/asr/runtime.js';
|
|
6
5
|
import type { CaptionInput } from './transcribe.js';
|
|
6
|
+
/** The installed private media helper and the release hash it must match before it runs. */
|
|
7
|
+
export interface MediaRuntime {
|
|
8
|
+
helper: string;
|
|
9
|
+
helperSha256: string;
|
|
10
|
+
}
|
|
7
11
|
export interface MediaCaptureRequest {
|
|
8
12
|
requestId: string;
|
|
9
13
|
sourceUrl: string;
|
|
10
|
-
|
|
14
|
+
mediaRuntime?: MediaRuntime;
|
|
11
15
|
stagingRoot: string;
|
|
12
16
|
connection?: CaptureConnection;
|
|
13
17
|
language?: string;
|
|
@@ -1,17 +1,22 @@
|
|
|
1
1
|
import { constants } from 'node:fs';
|
|
2
|
-
import { open, mkdir, mkdtemp, writeFile, statfs, realpath } from 'node:fs/promises';
|
|
2
|
+
import { open, mkdir, mkdtemp, readFile, writeFile, statfs, realpath } from 'node:fs/promises';
|
|
3
3
|
import { join, relative, isAbsolute } from 'node:path';
|
|
4
|
-
import { fileURLToPath } from 'node:url';
|
|
5
|
-
import { homedir } from 'node:os';
|
|
6
4
|
import { createHash } from 'node:crypto';
|
|
7
5
|
import { sha256 as hash } from '../util/hash.js';
|
|
8
6
|
import { limits } from '../../contracts/index.js';
|
|
9
7
|
import { failure } from '../acquisition/errors.js';
|
|
10
|
-
import { inspectCaptureRuntime } from '../acquisition/runtime.js';
|
|
11
8
|
import { privateRoot, physicalBytes, readConfined } from '../acquisition/files.js';
|
|
12
9
|
import { runOwned, isolatedEnvironment } from '../acquisition/process.js';
|
|
13
10
|
import { settleCaptureRoot } from '../acquisition/cleanup.js';
|
|
14
11
|
import { SpongeError, publicCode, codeOf } from '../errors.js';
|
|
12
|
+
import { nativePlatformSupported } from '../models/catalog.js';
|
|
13
|
+
async function verifyHelper(runtime) {
|
|
14
|
+
if (!isAbsolute(runtime.helper) || (await realpath(runtime.helper)) !== runtime.helper)
|
|
15
|
+
throw new SpongeError('integrity');
|
|
16
|
+
const data = await readFile(runtime.helper);
|
|
17
|
+
if (data.length > 32 * 1024 ** 2 || hash(data) !== runtime.helperSha256)
|
|
18
|
+
throw new SpongeError('integrity');
|
|
19
|
+
}
|
|
15
20
|
/** All ephemeral bytes remain below one capture owner; deletion requires a settled process tree. */
|
|
16
21
|
export async function captureMedia(request) {
|
|
17
22
|
const { requestId, signal } = request;
|
|
@@ -27,13 +32,16 @@ export async function captureMedia(request) {
|
|
|
27
32
|
return failure(requestId, 'invalid-input');
|
|
28
33
|
}
|
|
29
34
|
url.hash = '';
|
|
30
|
-
if (request.connection
|
|
35
|
+
if (request.connection)
|
|
31
36
|
return failure(requestId, 'unsupported');
|
|
32
37
|
if (signal?.aborted)
|
|
33
38
|
return failure(requestId, 'cancelled');
|
|
34
|
-
|
|
39
|
+
const runtime = request.mediaRuntime;
|
|
40
|
+
if (!runtime)
|
|
41
|
+
return failure(requestId, nativePlatformSupported() ? 'runtime-missing' : 'unsupported');
|
|
42
|
+
let base;
|
|
35
43
|
try {
|
|
36
|
-
|
|
44
|
+
await verifyHelper(runtime);
|
|
37
45
|
base = await privateRoot(request.stagingRoot);
|
|
38
46
|
const disk = await statfs(base);
|
|
39
47
|
if (disk.bavail * disk.bsize < limits.stagingBytes + limits.freeReserveBytes)
|
|
@@ -42,17 +50,6 @@ export async function captureMedia(request) {
|
|
|
42
50
|
catch (error) {
|
|
43
51
|
return failure(requestId, publicCode(error, 'invalid-input'));
|
|
44
52
|
}
|
|
45
|
-
let selectedState;
|
|
46
|
-
if (request.connection?.kind === 'account') {
|
|
47
|
-
if (!/^[A-Za-z0-9_.-]{1,128}$/.test(request.connection.accountId))
|
|
48
|
-
return failure(requestId, 'invalid-input');
|
|
49
|
-
try {
|
|
50
|
-
selectedState = await privateRoot(request.connection.stateHome);
|
|
51
|
-
}
|
|
52
|
-
catch {
|
|
53
|
-
return failure(requestId, 'repair-auth');
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
53
|
const root = await realpath(await mkdtemp(join(base, 'capture-'))), home = join(root, 'home');
|
|
57
54
|
let cleanup = 'settled', keep = false, checking = false, quota = false;
|
|
58
55
|
const controller = new AbortController(), abort = () => controller.abort();
|
|
@@ -84,28 +81,26 @@ export async function captureMedia(request) {
|
|
|
84
81
|
});
|
|
85
82
|
const requestFile = join(root, 'request.json');
|
|
86
83
|
await writeFile(requestFile, JSON.stringify({
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
language: request.language ?? 'en',
|
|
94
|
-
preferAudio: request.forceLocal === true,
|
|
95
|
-
maxBytes: 512 * 1024 ** 2,
|
|
96
|
-
maxDurationSeconds: 3600,
|
|
97
|
-
timeoutMs: limits.captureDeadlineMs,
|
|
98
|
-
},
|
|
84
|
+
url: url.href,
|
|
85
|
+
stagingParent: root,
|
|
86
|
+
language: request.language ?? 'en',
|
|
87
|
+
preferAudio: request.forceLocal === true,
|
|
88
|
+
maxBytes: 512 * 1024 ** 2,
|
|
89
|
+
maxDurationSeconds: 3600,
|
|
99
90
|
}), { mode: 0o600, flag: 'wx' });
|
|
100
|
-
const
|
|
101
|
-
const environment = isolatedEnvironment(home, home, selectedState ?? join(home, 'state'));
|
|
91
|
+
const environment = isolatedEnvironment(home, home);
|
|
102
92
|
environment.XDG_CACHE_HOME = home;
|
|
103
93
|
environment.XDG_CONFIG_HOME = home;
|
|
104
94
|
environment.XDG_STATE_HOME = home;
|
|
105
|
-
|
|
106
|
-
|
|
95
|
+
// yt-dlp, FFmpeg and yt-dlp's JavaScript runtime come from the user's PATH.
|
|
96
|
+
environment.PATH = [
|
|
97
|
+
...new Set([
|
|
98
|
+
...(process.env.PATH ?? '').split(':').filter((d) => isAbsolute(d)),
|
|
99
|
+
...environment.PATH.split(':'),
|
|
100
|
+
]),
|
|
101
|
+
].join(':');
|
|
107
102
|
cleanup = 'pending';
|
|
108
|
-
const result = await (request.runProcess ?? runOwned)(runtime.
|
|
103
|
+
const result = await (request.runProcess ?? runOwned)(runtime.helper, [requestFile], {
|
|
109
104
|
cwd: root,
|
|
110
105
|
env: environment,
|
|
111
106
|
signal: controller.signal,
|
|
@@ -121,22 +116,18 @@ export async function captureMedia(request) {
|
|
|
121
116
|
return failure(requestId, result.stopped);
|
|
122
117
|
const parsed = JSON.parse(result.stdout);
|
|
123
118
|
if (parsed.ok !== true) {
|
|
124
|
-
if (parsed.code === 'cleanup-required')
|
|
125
|
-
cleanup = 'pending';
|
|
126
119
|
const mapping = {
|
|
127
|
-
'permission-or-provider-failure': '
|
|
128
|
-
timeout: 'unavailable',
|
|
129
|
-
'cleanup-required': 'cleanup-required',
|
|
120
|
+
'permission-or-provider-failure': 'unavailable',
|
|
130
121
|
'invalid-input': 'invalid-input',
|
|
122
|
+
'tool-unavailable': 'runtime-missing',
|
|
131
123
|
unsupported: 'unsupported',
|
|
132
124
|
unavailable: 'unavailable',
|
|
133
125
|
'limit-exceeded': 'limit-exceeded',
|
|
134
|
-
cancelled: 'cancelled',
|
|
135
126
|
integrity: 'integrity',
|
|
136
127
|
};
|
|
137
128
|
return failure(requestId, mapping[parsed.code] ?? 'unclassified');
|
|
138
129
|
}
|
|
139
|
-
const manifest = parsed.manifest, mediaRoot = parsed.
|
|
130
|
+
const manifest = parsed.manifest, mediaRoot = parsed.root;
|
|
140
131
|
if (result.code !== 0 ||
|
|
141
132
|
typeof mediaRoot !== 'string' ||
|
|
142
133
|
!mediaRoot.startsWith(root + '/media-') ||
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ModelPackage } from './artifacts.js';
|
|
2
|
-
export type NativeKind = 'asr' | 'embedding';
|
|
3
|
-
export type NativeRole = 'asr-helper' | 'embedding-helper' | 'embedding-library' | 'notices';
|
|
2
|
+
export type NativeKind = 'asr' | 'embedding' | 'media';
|
|
3
|
+
export type NativeRole = 'asr-helper' | 'embedding-helper' | 'embedding-library' | 'media-helper' | 'notices';
|
|
4
4
|
export declare const nativeAssetNames: Record<NativeRole, string>;
|
|
5
5
|
interface NativeArtifact {
|
|
6
6
|
role: NativeRole;
|
|
@@ -19,8 +19,10 @@ export interface NativeCatalog {
|
|
|
19
19
|
build: {
|
|
20
20
|
asrSourceSha256: string;
|
|
21
21
|
embeddingSourceSha256: string;
|
|
22
|
+
mediaSourceSha256: string;
|
|
22
23
|
swiftVersion: string;
|
|
23
24
|
clangVersion: string;
|
|
25
|
+
rustcVersion: string;
|
|
24
26
|
};
|
|
25
27
|
}
|
|
26
28
|
/** Package-embedded release hashes are authority; local probe helper hashes never enter this catalog. */
|
|
@@ -10,6 +10,7 @@ export const nativeAssetNames = {
|
|
|
10
10
|
'asr-helper': 'sponge-speech-macos26-arm64',
|
|
11
11
|
'embedding-helper': 'sponge-embedding-macos26-arm64',
|
|
12
12
|
'embedding-library': 'libCLiteRTLM_mac.dylib',
|
|
13
|
+
'media-helper': 'sponge-media-macos26-arm64',
|
|
13
14
|
notices: 'native-NOTICES.txt',
|
|
14
15
|
};
|
|
15
16
|
const fail = () => {
|
|
@@ -43,7 +44,7 @@ export function parseNativeCatalog(value, expectedVersion) {
|
|
|
43
44
|
r.architecture !== 'arm64' ||
|
|
44
45
|
r.minimumMacOS !== '26.0' ||
|
|
45
46
|
!Array.isArray(r.artifacts) ||
|
|
46
|
-
r.artifacts.length !==
|
|
47
|
+
r.artifacts.length !== Object.keys(nativeAssetNames).length)
|
|
47
48
|
fail();
|
|
48
49
|
const roles = new Set();
|
|
49
50
|
const artifacts = r.artifacts.map((value) => {
|
|
@@ -64,10 +65,18 @@ export function parseNativeCatalog(value, expectedVersion) {
|
|
|
64
65
|
fail();
|
|
65
66
|
return a;
|
|
66
67
|
});
|
|
67
|
-
const build = object(r.build, [
|
|
68
|
+
const build = object(r.build, [
|
|
69
|
+
'asrSourceSha256',
|
|
70
|
+
'embeddingSourceSha256',
|
|
71
|
+
'mediaSourceSha256',
|
|
72
|
+
'swiftVersion',
|
|
73
|
+
'clangVersion',
|
|
74
|
+
'rustcVersion',
|
|
75
|
+
]);
|
|
68
76
|
if (!sha(build.asrSourceSha256) ||
|
|
69
77
|
!sha(build.embeddingSourceSha256) ||
|
|
70
|
-
|
|
78
|
+
!sha(build.mediaSourceSha256) ||
|
|
79
|
+
['swiftVersion', 'clangVersion', 'rustcVersion'].some((k) => typeof build[k] !== 'string' || !build[k] || Buffer.byteLength(String(build[k])) > 2048))
|
|
71
80
|
fail();
|
|
72
81
|
return { ...r, artifacts, build };
|
|
73
82
|
}
|
|
@@ -104,37 +113,47 @@ export async function loadNativeCatalog(expectedVersion, path = nativeCatalogPat
|
|
|
104
113
|
await file.close();
|
|
105
114
|
}
|
|
106
115
|
}
|
|
116
|
+
const helperNames = {
|
|
117
|
+
'asr-helper': 'sponge-speech',
|
|
118
|
+
'embedding-helper': 'sponge-embedding',
|
|
119
|
+
'media-helper': 'sponge-media',
|
|
120
|
+
};
|
|
107
121
|
export function nativeModelPackage(catalog, kind) {
|
|
108
122
|
parseNativeCatalog(catalog, catalog.packageVersion);
|
|
109
|
-
|
|
123
|
+
const roles = {
|
|
124
|
+
asr: ['asr-helper', 'notices'],
|
|
125
|
+
embedding: ['embedding-helper', 'embedding-library', 'notices'],
|
|
126
|
+
media: ['media-helper', 'notices'],
|
|
127
|
+
}[kind];
|
|
128
|
+
if (!roles)
|
|
110
129
|
fail();
|
|
111
|
-
const roles = kind === 'asr' ? ['asr-helper', 'notices'] : ['embedding-helper', 'embedding-library', 'notices'];
|
|
112
130
|
const artifacts = roles.map((role) => {
|
|
113
131
|
const a = catalog.artifacts.find((a) => a.role === role);
|
|
114
132
|
return {
|
|
115
|
-
name: role
|
|
133
|
+
name: role.endsWith('helper') ? helperNames[role] : a.name,
|
|
116
134
|
url: `https://github.com/hraness/sponge/releases/download/v${catalog.packageVersion}/${a.name}`,
|
|
117
135
|
sha256: a.sha256,
|
|
118
136
|
bytes: a.bytes,
|
|
119
137
|
...(role.endsWith('helper') ? { executable: true } : {}),
|
|
120
138
|
};
|
|
121
139
|
});
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
140
|
+
if (kind !== 'media')
|
|
141
|
+
artifacts.push(kind === 'asr'
|
|
142
|
+
? {
|
|
143
|
+
name: 'whistle.cact',
|
|
144
|
+
url: 'https://huggingface.co/Cactus-Compute/whistle/resolve/ca5287601bef25af26dcf2e1b2bdc0843a7c19e5/whistle.cact',
|
|
145
|
+
sha256: asrIdentity.modelSha256,
|
|
146
|
+
bytes: asrIdentity.modelBytes,
|
|
147
|
+
}
|
|
148
|
+
: {
|
|
149
|
+
name: 'embeddinggemma-2-text-vision-440m.litertlm',
|
|
150
|
+
url: 'https://huggingface.co/litert-community/embeddinggemma-2-text-vision-440m-litert-lm/resolve/e301f74d5551b0c2641bd5cb4652a76239d5c5f8/embeddinggemma-2-text-vision-440m.litertlm',
|
|
151
|
+
sha256: embeddingIdentity.modelSha256,
|
|
152
|
+
bytes: 387710976,
|
|
153
|
+
});
|
|
135
154
|
return {
|
|
136
155
|
id: `sponge-${kind}-${catalog.packageVersion}-macos26-arm64`,
|
|
137
|
-
license: 'Sponge helpers: proprietary; Needle, Whistle, LiteRT-LM and model: Apache-2.0; see native-NOTICES.txt',
|
|
156
|
+
license: 'Sponge helpers: proprietary; Needle, Whistle, LiteRT-LM and model: Apache-2.0; Rust crates: MIT or Apache-2.0; see native-NOTICES.txt',
|
|
138
157
|
artifacts,
|
|
139
158
|
};
|
|
140
159
|
}
|