@hraness/sponge 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -16
- package/dist/src/acquisition/capture.d.ts +1 -4
- package/dist/src/acquisition/capture.js +12 -23
- package/dist/src/acquisition/clip-helper.js +14 -0
- package/dist/src/acquisition/errors.d.ts +2 -1
- package/dist/src/acquisition/errors.js +4 -33
- package/dist/src/acquisition/lock.d.ts +1 -1
- package/dist/src/acquisition/lock.js +1 -1
- package/dist/src/acquisition/pins.d.ts +1 -1
- package/dist/src/acquisition/pins.js +3 -9
- package/dist/src/acquisition/process.d.ts +1 -1
- package/dist/src/acquisition/process.js +1 -2
- package/dist/src/acquisition/runtime.d.ts +0 -4
- package/dist/src/acquisition/runtime.js +12 -18
- package/dist/src/cli/commands.js +34 -16
- package/dist/src/cli/config.d.ts +2 -1
- package/dist/src/cli/config.js +8 -8
- package/dist/src/cli/runtimes.js +1 -1
- package/dist/src/cli/worker.js +13 -10
- package/dist/src/digest/pipeline.d.ts +29 -4
- package/dist/src/digest/pipeline.js +125 -21
- package/dist/src/errors.d.ts +2 -0
- package/dist/src/errors.js +4 -0
- package/dist/src/media/capture.d.ts +6 -2
- package/dist/src/media/capture.js +33 -42
- package/dist/src/models/catalog.d.ts +4 -2
- package/dist/src/models/catalog.js +39 -20
- package/dist/src/models/runtime.d.ts +11 -3
- package/dist/src/models/runtime.js +18 -11
- package/dist/src/pdf/capture-helper.js +6 -5
- package/dist/src/pdf/capture.d.ts +0 -4
- package/dist/src/pdf/capture.js +2 -14
- package/dist/src/pdf/errors.js +1 -3
- package/dist/src/store/digest-jobs.d.ts +5 -2
- package/dist/src/store/digest-jobs.js +7 -3
- package/dist/src/store/revisions.js +1 -1
- package/dist/src/worker/basic.d.ts +1 -1
- package/dist/src/worker/basic.js +5 -3
- package/native/runtime-manifest.json +14 -6
- package/package.json +1 -1
- package/dist/src/media/capture-helper.js +0 -21
- /package/dist/src/{media/capture-helper.d.ts → acquisition/clip-helper.d.ts} +0 -0
package/dist/src/errors.js
CHANGED
|
@@ -33,8 +33,10 @@ const parents = {
|
|
|
33
33
|
'draft-rejected': 'unavailable',
|
|
34
34
|
'model-unavailable': 'unavailable',
|
|
35
35
|
'model-busy': 'unavailable',
|
|
36
|
+
'source-too-thin': 'unavailable',
|
|
36
37
|
'parent-missing': 'invalid-input',
|
|
37
38
|
'target-exists': 'invalid-input',
|
|
39
|
+
'connection-retired': 'unsupported',
|
|
38
40
|
};
|
|
39
41
|
export const indexCodes = [
|
|
40
42
|
'invalid-index',
|
|
@@ -84,6 +86,8 @@ export const hints = {
|
|
|
84
86
|
export const detailHints = {
|
|
85
87
|
'draft-rejected': 'The model draft failed the digest checks listed above, even after one repair. Drafting is deterministic, so the same capture and model fail again; refresh the item or use another model.',
|
|
86
88
|
'model-unavailable': 'The model server did not answer or returned an error. Check that it is still running and has free GPU memory (another loaded model can exhaust it), then retry.',
|
|
89
|
+
'source-too-thin': 'The capture holds too little text to digest: under 20 words of prose, or only an archive’s metadata. Refresh the item when a fuller capture is possible.',
|
|
90
|
+
'connection-retired': 'Saved GhostGet account connections were retired in Sponge 0.5.0. Attach a signed-in browser over CDP and add it to connections.json instead.',
|
|
87
91
|
'model-busy': 'Another llama-server is running, and two loaded models can exhaust GPU memory. Wait for it to exit, then retry.',
|
|
88
92
|
};
|
|
89
93
|
/** The one error type. `message` is the code, so bare `String(error)` never carries detail, paths or causes. */
|
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
import type { Asset, ErrorEnvelope } from '../../contracts/index.js';
|
|
2
|
-
import { type CaptureRuntime } from '../acquisition/runtime.js';
|
|
3
2
|
import { runOwned } from '../acquisition/process.js';
|
|
4
3
|
import type { CaptureConnection } from '../acquisition/index.js';
|
|
5
4
|
import type { AudioInput } from '../inference/asr/runtime.js';
|
|
6
5
|
import type { CaptionInput } from './transcribe.js';
|
|
6
|
+
/** The installed private media helper and the release hash it must match before it runs. */
|
|
7
|
+
export interface MediaRuntime {
|
|
8
|
+
helper: string;
|
|
9
|
+
helperSha256: string;
|
|
10
|
+
}
|
|
7
11
|
export interface MediaCaptureRequest {
|
|
8
12
|
requestId: string;
|
|
9
13
|
sourceUrl: string;
|
|
10
|
-
|
|
14
|
+
mediaRuntime?: MediaRuntime;
|
|
11
15
|
stagingRoot: string;
|
|
12
16
|
connection?: CaptureConnection;
|
|
13
17
|
language?: string;
|
|
@@ -1,17 +1,22 @@
|
|
|
1
1
|
import { constants } from 'node:fs';
|
|
2
|
-
import { open, mkdir, mkdtemp, writeFile, statfs, realpath } from 'node:fs/promises';
|
|
2
|
+
import { open, mkdir, mkdtemp, readFile, writeFile, statfs, realpath } from 'node:fs/promises';
|
|
3
3
|
import { join, relative, isAbsolute } from 'node:path';
|
|
4
|
-
import { fileURLToPath } from 'node:url';
|
|
5
|
-
import { homedir } from 'node:os';
|
|
6
4
|
import { createHash } from 'node:crypto';
|
|
7
5
|
import { sha256 as hash } from '../util/hash.js';
|
|
8
6
|
import { limits } from '../../contracts/index.js';
|
|
9
7
|
import { failure } from '../acquisition/errors.js';
|
|
10
|
-
import { inspectCaptureRuntime } from '../acquisition/runtime.js';
|
|
11
8
|
import { privateRoot, physicalBytes, readConfined } from '../acquisition/files.js';
|
|
12
9
|
import { runOwned, isolatedEnvironment } from '../acquisition/process.js';
|
|
13
10
|
import { settleCaptureRoot } from '../acquisition/cleanup.js';
|
|
14
11
|
import { SpongeError, publicCode, codeOf } from '../errors.js';
|
|
12
|
+
import { nativePlatformSupported } from '../models/catalog.js';
|
|
13
|
+
async function verifyHelper(runtime) {
|
|
14
|
+
if (!isAbsolute(runtime.helper) || (await realpath(runtime.helper)) !== runtime.helper)
|
|
15
|
+
throw new SpongeError('integrity');
|
|
16
|
+
const data = await readFile(runtime.helper);
|
|
17
|
+
if (data.length > 32 * 1024 ** 2 || hash(data) !== runtime.helperSha256)
|
|
18
|
+
throw new SpongeError('integrity');
|
|
19
|
+
}
|
|
15
20
|
/** All ephemeral bytes remain below one capture owner; deletion requires a settled process tree. */
|
|
16
21
|
export async function captureMedia(request) {
|
|
17
22
|
const { requestId, signal } = request;
|
|
@@ -27,13 +32,16 @@ export async function captureMedia(request) {
|
|
|
27
32
|
return failure(requestId, 'invalid-input');
|
|
28
33
|
}
|
|
29
34
|
url.hash = '';
|
|
30
|
-
if (request.connection
|
|
35
|
+
if (request.connection)
|
|
31
36
|
return failure(requestId, 'unsupported');
|
|
32
37
|
if (signal?.aborted)
|
|
33
38
|
return failure(requestId, 'cancelled');
|
|
34
|
-
|
|
39
|
+
const runtime = request.mediaRuntime;
|
|
40
|
+
if (!runtime)
|
|
41
|
+
return failure(requestId, nativePlatformSupported() ? 'runtime-missing' : 'unsupported');
|
|
42
|
+
let base;
|
|
35
43
|
try {
|
|
36
|
-
|
|
44
|
+
await verifyHelper(runtime);
|
|
37
45
|
base = await privateRoot(request.stagingRoot);
|
|
38
46
|
const disk = await statfs(base);
|
|
39
47
|
if (disk.bavail * disk.bsize < limits.stagingBytes + limits.freeReserveBytes)
|
|
@@ -42,17 +50,6 @@ export async function captureMedia(request) {
|
|
|
42
50
|
catch (error) {
|
|
43
51
|
return failure(requestId, publicCode(error, 'invalid-input'));
|
|
44
52
|
}
|
|
45
|
-
let selectedState;
|
|
46
|
-
if (request.connection?.kind === 'account') {
|
|
47
|
-
if (!/^[A-Za-z0-9_.-]{1,128}$/.test(request.connection.accountId))
|
|
48
|
-
return failure(requestId, 'invalid-input');
|
|
49
|
-
try {
|
|
50
|
-
selectedState = await privateRoot(request.connection.stateHome);
|
|
51
|
-
}
|
|
52
|
-
catch {
|
|
53
|
-
return failure(requestId, 'repair-auth');
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
53
|
const root = await realpath(await mkdtemp(join(base, 'capture-'))), home = join(root, 'home');
|
|
57
54
|
let cleanup = 'settled', keep = false, checking = false, quota = false;
|
|
58
55
|
const controller = new AbortController(), abort = () => controller.abort();
|
|
@@ -84,28 +81,26 @@ export async function captureMedia(request) {
|
|
|
84
81
|
});
|
|
85
82
|
const requestFile = join(root, 'request.json');
|
|
86
83
|
await writeFile(requestFile, JSON.stringify({
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
language: request.language ?? 'en',
|
|
94
|
-
preferAudio: request.forceLocal === true,
|
|
95
|
-
maxBytes: 512 * 1024 ** 2,
|
|
96
|
-
maxDurationSeconds: 3600,
|
|
97
|
-
timeoutMs: limits.captureDeadlineMs,
|
|
98
|
-
},
|
|
84
|
+
url: url.href,
|
|
85
|
+
stagingParent: root,
|
|
86
|
+
language: request.language ?? 'en',
|
|
87
|
+
preferAudio: request.forceLocal === true,
|
|
88
|
+
maxBytes: 512 * 1024 ** 2,
|
|
89
|
+
maxDurationSeconds: 3600,
|
|
99
90
|
}), { mode: 0o600, flag: 'wx' });
|
|
100
|
-
const
|
|
101
|
-
const environment = isolatedEnvironment(home, home, selectedState ?? join(home, 'state'));
|
|
91
|
+
const environment = isolatedEnvironment(home, home);
|
|
102
92
|
environment.XDG_CACHE_HOME = home;
|
|
103
93
|
environment.XDG_CONFIG_HOME = home;
|
|
104
94
|
environment.XDG_STATE_HOME = home;
|
|
105
|
-
|
|
106
|
-
|
|
95
|
+
// yt-dlp, FFmpeg and yt-dlp's JavaScript runtime come from the user's PATH.
|
|
96
|
+
environment.PATH = [
|
|
97
|
+
...new Set([
|
|
98
|
+
...(process.env.PATH ?? '').split(':').filter((d) => isAbsolute(d)),
|
|
99
|
+
...environment.PATH.split(':'),
|
|
100
|
+
]),
|
|
101
|
+
].join(':');
|
|
107
102
|
cleanup = 'pending';
|
|
108
|
-
const result = await (request.runProcess ?? runOwned)(runtime.
|
|
103
|
+
const result = await (request.runProcess ?? runOwned)(runtime.helper, [requestFile], {
|
|
109
104
|
cwd: root,
|
|
110
105
|
env: environment,
|
|
111
106
|
signal: controller.signal,
|
|
@@ -121,22 +116,18 @@ export async function captureMedia(request) {
|
|
|
121
116
|
return failure(requestId, result.stopped);
|
|
122
117
|
const parsed = JSON.parse(result.stdout);
|
|
123
118
|
if (parsed.ok !== true) {
|
|
124
|
-
if (parsed.code === 'cleanup-required')
|
|
125
|
-
cleanup = 'pending';
|
|
126
119
|
const mapping = {
|
|
127
|
-
'permission-or-provider-failure': '
|
|
128
|
-
timeout: 'unavailable',
|
|
129
|
-
'cleanup-required': 'cleanup-required',
|
|
120
|
+
'permission-or-provider-failure': 'unavailable',
|
|
130
121
|
'invalid-input': 'invalid-input',
|
|
122
|
+
'tool-unavailable': 'runtime-missing',
|
|
131
123
|
unsupported: 'unsupported',
|
|
132
124
|
unavailable: 'unavailable',
|
|
133
125
|
'limit-exceeded': 'limit-exceeded',
|
|
134
|
-
cancelled: 'cancelled',
|
|
135
126
|
integrity: 'integrity',
|
|
136
127
|
};
|
|
137
128
|
return failure(requestId, mapping[parsed.code] ?? 'unclassified');
|
|
138
129
|
}
|
|
139
|
-
const manifest = parsed.manifest, mediaRoot = parsed.
|
|
130
|
+
const manifest = parsed.manifest, mediaRoot = parsed.root;
|
|
140
131
|
if (result.code !== 0 ||
|
|
141
132
|
typeof mediaRoot !== 'string' ||
|
|
142
133
|
!mediaRoot.startsWith(root + '/media-') ||
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ModelPackage } from './artifacts.js';
|
|
2
|
-
export type NativeKind = 'asr' | 'embedding';
|
|
3
|
-
export type NativeRole = 'asr-helper' | 'embedding-helper' | 'embedding-library' | 'notices';
|
|
2
|
+
export type NativeKind = 'asr' | 'embedding' | 'media';
|
|
3
|
+
export type NativeRole = 'asr-helper' | 'embedding-helper' | 'embedding-library' | 'media-helper' | 'notices';
|
|
4
4
|
export declare const nativeAssetNames: Record<NativeRole, string>;
|
|
5
5
|
interface NativeArtifact {
|
|
6
6
|
role: NativeRole;
|
|
@@ -19,8 +19,10 @@ export interface NativeCatalog {
|
|
|
19
19
|
build: {
|
|
20
20
|
asrSourceSha256: string;
|
|
21
21
|
embeddingSourceSha256: string;
|
|
22
|
+
mediaSourceSha256: string;
|
|
22
23
|
swiftVersion: string;
|
|
23
24
|
clangVersion: string;
|
|
25
|
+
rustcVersion: string;
|
|
24
26
|
};
|
|
25
27
|
}
|
|
26
28
|
/** Package-embedded release hashes are authority; local probe helper hashes never enter this catalog. */
|
|
@@ -10,6 +10,7 @@ export const nativeAssetNames = {
|
|
|
10
10
|
'asr-helper': 'sponge-speech-macos26-arm64',
|
|
11
11
|
'embedding-helper': 'sponge-embedding-macos26-arm64',
|
|
12
12
|
'embedding-library': 'libCLiteRTLM_mac.dylib',
|
|
13
|
+
'media-helper': 'sponge-media-macos26-arm64',
|
|
13
14
|
notices: 'native-NOTICES.txt',
|
|
14
15
|
};
|
|
15
16
|
const fail = () => {
|
|
@@ -43,7 +44,7 @@ export function parseNativeCatalog(value, expectedVersion) {
|
|
|
43
44
|
r.architecture !== 'arm64' ||
|
|
44
45
|
r.minimumMacOS !== '26.0' ||
|
|
45
46
|
!Array.isArray(r.artifacts) ||
|
|
46
|
-
r.artifacts.length !==
|
|
47
|
+
r.artifacts.length !== Object.keys(nativeAssetNames).length)
|
|
47
48
|
fail();
|
|
48
49
|
const roles = new Set();
|
|
49
50
|
const artifacts = r.artifacts.map((value) => {
|
|
@@ -64,10 +65,18 @@ export function parseNativeCatalog(value, expectedVersion) {
|
|
|
64
65
|
fail();
|
|
65
66
|
return a;
|
|
66
67
|
});
|
|
67
|
-
const build = object(r.build, [
|
|
68
|
+
const build = object(r.build, [
|
|
69
|
+
'asrSourceSha256',
|
|
70
|
+
'embeddingSourceSha256',
|
|
71
|
+
'mediaSourceSha256',
|
|
72
|
+
'swiftVersion',
|
|
73
|
+
'clangVersion',
|
|
74
|
+
'rustcVersion',
|
|
75
|
+
]);
|
|
68
76
|
if (!sha(build.asrSourceSha256) ||
|
|
69
77
|
!sha(build.embeddingSourceSha256) ||
|
|
70
|
-
|
|
78
|
+
!sha(build.mediaSourceSha256) ||
|
|
79
|
+
['swiftVersion', 'clangVersion', 'rustcVersion'].some((k) => typeof build[k] !== 'string' || !build[k] || Buffer.byteLength(String(build[k])) > 2048))
|
|
71
80
|
fail();
|
|
72
81
|
return { ...r, artifacts, build };
|
|
73
82
|
}
|
|
@@ -104,37 +113,47 @@ export async function loadNativeCatalog(expectedVersion, path = nativeCatalogPat
|
|
|
104
113
|
await file.close();
|
|
105
114
|
}
|
|
106
115
|
}
|
|
116
|
+
const helperNames = {
|
|
117
|
+
'asr-helper': 'sponge-speech',
|
|
118
|
+
'embedding-helper': 'sponge-embedding',
|
|
119
|
+
'media-helper': 'sponge-media',
|
|
120
|
+
};
|
|
107
121
|
export function nativeModelPackage(catalog, kind) {
|
|
108
122
|
parseNativeCatalog(catalog, catalog.packageVersion);
|
|
109
|
-
|
|
123
|
+
const roles = {
|
|
124
|
+
asr: ['asr-helper', 'notices'],
|
|
125
|
+
embedding: ['embedding-helper', 'embedding-library', 'notices'],
|
|
126
|
+
media: ['media-helper', 'notices'],
|
|
127
|
+
}[kind];
|
|
128
|
+
if (!roles)
|
|
110
129
|
fail();
|
|
111
|
-
const roles = kind === 'asr' ? ['asr-helper', 'notices'] : ['embedding-helper', 'embedding-library', 'notices'];
|
|
112
130
|
const artifacts = roles.map((role) => {
|
|
113
131
|
const a = catalog.artifacts.find((a) => a.role === role);
|
|
114
132
|
return {
|
|
115
|
-
name: role
|
|
133
|
+
name: role.endsWith('helper') ? helperNames[role] : a.name,
|
|
116
134
|
url: `https://github.com/hraness/sponge/releases/download/v${catalog.packageVersion}/${a.name}`,
|
|
117
135
|
sha256: a.sha256,
|
|
118
136
|
bytes: a.bytes,
|
|
119
137
|
...(role.endsWith('helper') ? { executable: true } : {}),
|
|
120
138
|
};
|
|
121
139
|
});
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
140
|
+
if (kind !== 'media')
|
|
141
|
+
artifacts.push(kind === 'asr'
|
|
142
|
+
? {
|
|
143
|
+
name: 'whistle.cact',
|
|
144
|
+
url: 'https://huggingface.co/Cactus-Compute/whistle/resolve/ca5287601bef25af26dcf2e1b2bdc0843a7c19e5/whistle.cact',
|
|
145
|
+
sha256: asrIdentity.modelSha256,
|
|
146
|
+
bytes: asrIdentity.modelBytes,
|
|
147
|
+
}
|
|
148
|
+
: {
|
|
149
|
+
name: 'embeddinggemma-2-text-vision-440m.litertlm',
|
|
150
|
+
url: 'https://huggingface.co/litert-community/embeddinggemma-2-text-vision-440m-litert-lm/resolve/e301f74d5551b0c2641bd5cb4652a76239d5c5f8/embeddinggemma-2-text-vision-440m.litertlm',
|
|
151
|
+
sha256: embeddingIdentity.modelSha256,
|
|
152
|
+
bytes: 387710976,
|
|
153
|
+
});
|
|
135
154
|
return {
|
|
136
155
|
id: `sponge-${kind}-${catalog.packageVersion}-macos26-arm64`,
|
|
137
|
-
license: 'Sponge helpers: proprietary; Needle, Whistle, LiteRT-LM and model: Apache-2.0; see native-NOTICES.txt',
|
|
156
|
+
license: 'Sponge helpers: proprietary; Needle, Whistle, LiteRT-LM and model: Apache-2.0; Rust crates: MIT or Apache-2.0; see native-NOTICES.txt',
|
|
138
157
|
artifacts,
|
|
139
158
|
};
|
|
140
159
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { AsrRuntime } from '../inference/asr/runtime.js';
|
|
2
2
|
import type { EmbeddingRuntime } from '../inference/embedding/index.js';
|
|
3
|
+
import type { MediaRuntime } from '../media/capture.js';
|
|
3
4
|
import { type NativeCatalog, type NativeKind } from './catalog.js';
|
|
4
5
|
export declare function nativeSetupPlan(kind: NativeKind, version: string): Promise<{
|
|
5
6
|
supported: boolean;
|
|
@@ -11,12 +12,19 @@ export declare function nativeSetupPlan(kind: NativeKind, version: string): Prom
|
|
|
11
12
|
requiredFreeBytes: number;
|
|
12
13
|
artifacts: readonly import("./artifacts.js").ModelArtifact[];
|
|
13
14
|
}>;
|
|
15
|
+
type NativeRuntime = AsrRuntime | EmbeddingRuntime | MediaRuntime;
|
|
14
16
|
export declare function setupNativeRuntime(cacheRoot: string, catalog: NativeCatalog, kind: NativeKind, options?: {
|
|
15
17
|
signal?: AbortSignal;
|
|
16
|
-
}): Promise<
|
|
18
|
+
}): Promise<NativeRuntime>;
|
|
17
19
|
export declare function provisionNative(kind: 'asr', cacheRoot: string, version: string, signal?: AbortSignal): Promise<AsrRuntime>;
|
|
18
20
|
export declare function provisionNative(kind: 'embedding', cacheRoot: string, version: string, signal?: AbortSignal): Promise<EmbeddingRuntime>;
|
|
19
|
-
export declare function provisionNative(kind:
|
|
21
|
+
export declare function provisionNative(kind: 'media', cacheRoot: string, version: string, signal?: AbortSignal): Promise<MediaRuntime>;
|
|
22
|
+
export declare function provisionNative(kind: NativeKind, cacheRoot: string, version: string, signal?: AbortSignal): Promise<NativeRuntime>;
|
|
20
23
|
export declare function inspectNative(kind: 'asr', cacheRoot: string, version: string): Promise<AsrRuntime>;
|
|
21
24
|
export declare function inspectNative(kind: 'embedding', cacheRoot: string, version: string): Promise<EmbeddingRuntime>;
|
|
22
|
-
export declare function inspectNative(kind:
|
|
25
|
+
export declare function inspectNative(kind: 'media', cacheRoot: string, version: string): Promise<MediaRuntime>;
|
|
26
|
+
export declare function inspectNative(kind: NativeKind, cacheRoot: string, version: string): Promise<NativeRuntime>;
|
|
27
|
+
type RuntimeOf<K extends NativeKind> = K extends 'asr' ? AsrRuntime : K extends 'embedding' ? EmbeddingRuntime : MediaRuntime;
|
|
28
|
+
/** An installed runtime, or undefined when it is not installed or this platform cannot run it. */
|
|
29
|
+
export declare function optionalNative<K extends NativeKind>(kind: K, cacheRoot: string, version: string): Promise<RuntimeOf<K> | undefined>;
|
|
30
|
+
export {};
|
|
@@ -4,7 +4,7 @@ import { privateRoot } from '../acquisition/files.js';
|
|
|
4
4
|
import { nativeArtifactFetch } from './download.js';
|
|
5
5
|
import { inspectInstalledModelPackage, provisionModelPackage, recoverModelPackage, modelSetupPlan, } from './artifacts.js';
|
|
6
6
|
import { nativePlatformSupported, nativeModelPackage, loadNativeCatalog, } from './catalog.js';
|
|
7
|
-
import { SpongeError } from '../errors.js';
|
|
7
|
+
import { SpongeError, publicCode } from '../errors.js';
|
|
8
8
|
export async function nativeSetupPlan(kind, version) {
|
|
9
9
|
const catalog = await loadNativeCatalog(version);
|
|
10
10
|
return {
|
|
@@ -24,6 +24,10 @@ function asr(c, p) {
|
|
|
24
24
|
model: p['whistle.cact'],
|
|
25
25
|
};
|
|
26
26
|
}
|
|
27
|
+
function media(c, p) {
|
|
28
|
+
return { helper: p['sponge-media'], helperSha256: c.artifacts.find((a) => a.role === 'media-helper').sha256 };
|
|
29
|
+
}
|
|
30
|
+
const runtimeOf = (kind, c, p) => kind === 'asr' ? asr(c, p) : kind === 'embedding' ? embedding(c, p) : media(c, p);
|
|
27
31
|
function embedding(c, p) {
|
|
28
32
|
return {
|
|
29
33
|
helper: p['sponge-embedding'],
|
|
@@ -48,7 +52,7 @@ export async function setupNativeRuntime(cacheRoot, catalog, kind, options = {})
|
|
|
48
52
|
await privateRoot(cacheRoot);
|
|
49
53
|
await recoverModelPackage(cacheRoot, manifest);
|
|
50
54
|
const paths = await provisionModelPackage(cacheRoot, manifest, { ...options, fetch: nativeArtifactFetch(manifest) });
|
|
51
|
-
return kind
|
|
55
|
+
return runtimeOf(kind, catalog, paths);
|
|
52
56
|
}
|
|
53
57
|
async function installed(cacheRoot, catalog, kind) {
|
|
54
58
|
const paths = await inspectInstalledModelPackage(cacheRoot, nativeModelPackage(catalog, kind));
|
|
@@ -56,18 +60,21 @@ async function installed(cacheRoot, catalog, kind) {
|
|
|
56
60
|
throw new SpongeError('runtime-missing');
|
|
57
61
|
return paths;
|
|
58
62
|
}
|
|
59
|
-
async function inspectAsrRuntime(cacheRoot, catalog) {
|
|
60
|
-
supported();
|
|
61
|
-
return asr(catalog, await installed(cacheRoot, catalog, 'asr'));
|
|
62
|
-
}
|
|
63
|
-
async function inspectEmbeddingRuntime(cacheRoot, catalog) {
|
|
64
|
-
supported();
|
|
65
|
-
return embedding(catalog, await installed(cacheRoot, catalog, 'embedding'));
|
|
66
|
-
}
|
|
67
63
|
export async function provisionNative(kind, cacheRoot, version, signal) {
|
|
68
64
|
return setupNativeRuntime(cacheRoot, await loadNativeCatalog(version), kind, { signal });
|
|
69
65
|
}
|
|
70
66
|
export async function inspectNative(kind, cacheRoot, version) {
|
|
67
|
+
supported();
|
|
71
68
|
const catalog = await loadNativeCatalog(version);
|
|
72
|
-
return kind
|
|
69
|
+
return runtimeOf(kind, catalog, await installed(cacheRoot, catalog, kind));
|
|
70
|
+
}
|
|
71
|
+
/** An installed runtime, or undefined when it is not installed or this platform cannot run it. */
|
|
72
|
+
export async function optionalNative(kind, cacheRoot, version) {
|
|
73
|
+
try {
|
|
74
|
+
return (await inspectNative(kind, cacheRoot, version));
|
|
75
|
+
}
|
|
76
|
+
catch (error) {
|
|
77
|
+
if (!['runtime-missing', 'unsupported'].includes(publicCode(error, 'integrity')))
|
|
78
|
+
throw error;
|
|
79
|
+
}
|
|
73
80
|
}
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
// Pinned public
|
|
1
|
+
// Pinned public Wordcell bridge. Native subprocesses inherit the supervisor's group.
|
|
2
2
|
const runtime = process.argv[2], args = process.argv.slice(3);
|
|
3
3
|
try {
|
|
4
|
-
const {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
4
|
+
const { runPdfCommand, runPdfToolCommand } = await import(import.meta.resolve('@hraness/wordcell/pdf', `${runtime}/package.json`));
|
|
5
|
+
process.exitCode = await runPdfCommand(args, process.env, { stdout: (value) => process.stdout.write(value), stderr: (value) => process.stderr.write(value) }, {
|
|
6
|
+
captureDependencies: {
|
|
7
|
+
runTool: (specification) => runPdfToolCommand({ ...specification, processGroup: 'inherit' }),
|
|
8
|
+
},
|
|
8
9
|
});
|
|
9
10
|
}
|
|
10
11
|
catch {
|
|
@@ -2,10 +2,6 @@ import type { CaptureResult, ErrorEnvelope } from '../../contracts/index.js';
|
|
|
2
2
|
import { runOwned } from '../acquisition/process.js';
|
|
3
3
|
import { type CaptureRuntime } from '../acquisition/runtime.js';
|
|
4
4
|
type CaptureConnection = {
|
|
5
|
-
kind: 'account';
|
|
6
|
-
stateHome: string;
|
|
7
|
-
accountId: string;
|
|
8
|
-
} | {
|
|
9
5
|
kind: 'cdp';
|
|
10
6
|
endpoint: string;
|
|
11
7
|
};
|
package/dist/src/pdf/capture.js
CHANGED
|
@@ -52,7 +52,7 @@ export async function capturePdf(request) {
|
|
|
52
52
|
return failure(requestId, 'invalid-input');
|
|
53
53
|
if (url.protocol === 'file:' && (url.host || url.search || url.hash || request.connection))
|
|
54
54
|
return failure(requestId, 'invalid-input');
|
|
55
|
-
if (request.connection
|
|
55
|
+
if (request.connection || request.allowPrivateNetwork)
|
|
56
56
|
return failure(requestId, 'unsupported');
|
|
57
57
|
const captureHelper = fileURLToPath(new URL(import.meta.url.endsWith('.ts') ? './capture-helper.ts' : './capture-helper.js', import.meta.url));
|
|
58
58
|
const args = [
|
|
@@ -76,21 +76,9 @@ export async function capturePdf(request) {
|
|
|
76
76
|
'--quiet',
|
|
77
77
|
'--json',
|
|
78
78
|
];
|
|
79
|
-
let selectedState;
|
|
80
|
-
if (request.connection?.kind === 'account') {
|
|
81
|
-
if (!/^[a-zA-Z0-9_.-]{1,128}$/.test(request.connection.accountId))
|
|
82
|
-
return failure(requestId, 'invalid-input');
|
|
83
|
-
try {
|
|
84
|
-
selectedState = await privateRoot(request.connection.stateHome);
|
|
85
|
-
}
|
|
86
|
-
catch {
|
|
87
|
-
return failure(requestId, 'repair-auth');
|
|
88
|
-
}
|
|
89
|
-
args.push('--auth', request.connection.accountId);
|
|
90
|
-
}
|
|
91
79
|
const root = await realpath(await mkdtemp(join(base, 'capture-'))), home = join(root, 'home'), output = join(root, 'bundles');
|
|
92
80
|
args.push('--output', output);
|
|
93
|
-
const env = isolatedEnvironment(home, home
|
|
81
|
+
const env = isolatedEnvironment(home, home);
|
|
94
82
|
const controller = new AbortController();
|
|
95
83
|
const abort = () => controller.abort();
|
|
96
84
|
signal?.addEventListener('abort', abort, { once: true });
|
package/dist/src/pdf/errors.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { failure
|
|
1
|
+
import { failure } from '../acquisition/errors.js';
|
|
2
2
|
/** Wordcell's pinned PDF code field is authoritative; prose is never classified. */
|
|
3
3
|
export function pdfFailure(value, requestId) {
|
|
4
4
|
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
@@ -6,8 +6,6 @@ export function pdfFailure(value, requestId) {
|
|
|
6
6
|
const record = value;
|
|
7
7
|
if (record.ok !== false)
|
|
8
8
|
return failure(requestId, 'unclassified');
|
|
9
|
-
if (record.captureError)
|
|
10
|
-
return upstreamFailure(value, requestId);
|
|
11
9
|
const codes = {
|
|
12
10
|
'invalid-input': 'invalid-input',
|
|
13
11
|
encrypted: 'unsupported',
|
|
@@ -20,8 +20,11 @@ export type DigestOutcome = {
|
|
|
20
20
|
notBefore?: string;
|
|
21
21
|
countAttempt?: boolean;
|
|
22
22
|
};
|
|
23
|
-
/**
|
|
24
|
-
|
|
23
|
+
/**
|
|
24
|
+
* Queue a digest of a newly captured revision when auto-digest is on and the capture has enough text to digest. Runs
|
|
25
|
+
* inside the capture commit's transaction.
|
|
26
|
+
*/
|
|
27
|
+
export declare function enqueueDigest(db: DatabaseSync, itemId: string, revisionId: string, markdown: string): void;
|
|
25
28
|
/**
|
|
26
29
|
* Digest jobs under the shared worker lease. A job names one revision; when its item moves on or is deleted before the
|
|
27
30
|
* worker reaches it, the job is cancelled as `superseded`, since the newer revision has a job of its own. The drafting
|
|
@@ -3,12 +3,16 @@ import { text } from './files.js';
|
|
|
3
3
|
import { checkOwner } from './lease.js';
|
|
4
4
|
import { now, transaction } from './context.js';
|
|
5
5
|
import { SpongeError } from '../errors.js';
|
|
6
|
+
import { undigestible } from '../digest/pipeline.js';
|
|
6
7
|
/** Automatic retries a retryable failure gets before it stays failed. */
|
|
7
8
|
const maxAttempts = 3;
|
|
8
9
|
const alias = /^[\w.:/@+-]{1,256}$/;
|
|
9
|
-
/**
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
/**
|
|
11
|
+
* Queue a digest of a newly captured revision when auto-digest is on and the capture has enough text to digest. Runs
|
|
12
|
+
* inside the capture commit's transaction.
|
|
13
|
+
*/
|
|
14
|
+
export function enqueueDigest(db, itemId, revisionId, markdown) {
|
|
15
|
+
if (db.prepare('SELECT auto FROM digest_settings WHERE id=1').get()?.auto !== 1 || undigestible(markdown))
|
|
12
16
|
return;
|
|
13
17
|
db.prepare("INSERT INTO digest_jobs(id,item_id,revision_id,state,created_at) VALUES(?,?,?,'queued',?) ON CONFLICT(revision_id) DO NOTHING").run(randomUUID(), itemId, revisionId, now());
|
|
14
18
|
}
|
|
@@ -154,7 +154,7 @@ export class Revisions {
|
|
|
154
154
|
trackLibrary(db);
|
|
155
155
|
// A capture, not an import: imported libraries are digested on purpose, not as a side effect.
|
|
156
156
|
if (!imported)
|
|
157
|
-
enqueueDigest(db, input.itemId, revisionId);
|
|
157
|
+
enqueueDigest(db, input.itemId, revisionId, String(stored.markdown));
|
|
158
158
|
}
|
|
159
159
|
if (imported) {
|
|
160
160
|
const receipt = importReceipt(db, imported.namespace, imported.sourceKey, imported.sourceHash);
|
|
@@ -2,7 +2,7 @@ import type { LibraryStore } from '../store/index.js';
|
|
|
2
2
|
import { type ArticleOptions, type ArticleOutcome } from '../app/article.js';
|
|
3
3
|
import { type MediaOptions } from '../media/handler.js';
|
|
4
4
|
/** Content handlers over the durable sequential worker and its process supervisor. */
|
|
5
|
-
export declare function drainLibrary(store: LibraryStore, options: ArticleOptions & Pick<MediaOptions, 'asrRuntime' | 'language' | 'forceLocal'> & {
|
|
5
|
+
export declare function drainLibrary(store: LibraryStore, options: ArticleOptions & Pick<MediaOptions, 'asrRuntime' | 'mediaRuntime' | 'language' | 'forceLocal'> & {
|
|
6
6
|
jobId?: string;
|
|
7
7
|
resolveConnection?: (realm: string) => ArticleOptions['connection'];
|
|
8
8
|
onResult?: (result: ArticleOutcome) => void;
|
package/dist/src/worker/basic.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { failure } from '../acquisition/errors.js';
|
|
2
|
+
import { codeOf, publicCode } from '../errors.js';
|
|
2
3
|
import { ingestArticle } from '../app/article.js';
|
|
3
4
|
import { ingestPdf } from '../pdf/index.js';
|
|
4
5
|
import { ingestMedia } from '../media/handler.js';
|
|
@@ -11,10 +12,11 @@ export async function drainLibrary(store, options) {
|
|
|
11
12
|
try {
|
|
12
13
|
connection = options.resolveConnection?.(source?.realm ?? 'public') ?? options.connection;
|
|
13
14
|
}
|
|
14
|
-
catch {
|
|
15
|
-
|
|
15
|
+
catch (error) {
|
|
16
|
+
const code = codeOf(error) === 'connection-retired' ? 'connection-retired' : 'repair-auth';
|
|
17
|
+
store.queue.fail(context.attempt, code);
|
|
16
18
|
store.queue.settle(context.attempt);
|
|
17
|
-
return { ...failure(context.attempt.attemptId,
|
|
19
|
+
return { ...failure(context.attempt.attemptId, publicCode(code), 'acquisition'), cleanup: 'settled' };
|
|
18
20
|
}
|
|
19
21
|
if (source?.kind === 'media') {
|
|
20
22
|
const { capture: _articleCapture, ...mediaOptions } = options;
|
|
@@ -20,21 +20,29 @@
|
|
|
20
20
|
"sha256": "9526db9ee35e7eb4a5a58ac1e53330032f22a13890bd4ee01818fb380a57a362"
|
|
21
21
|
},
|
|
22
22
|
{
|
|
23
|
-
"bytes":
|
|
23
|
+
"bytes": 534912,
|
|
24
|
+
"name": "sponge-media-macos26-arm64",
|
|
25
|
+
"role": "media-helper",
|
|
26
|
+
"sha256": "4e01a12d163abcf8489feaf87e8050bcc1c260dcc39c6318f0bb7b1bbfffdc67"
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"bytes": 23606,
|
|
24
30
|
"name": "native-NOTICES.txt",
|
|
25
31
|
"role": "notices",
|
|
26
|
-
"sha256": "
|
|
32
|
+
"sha256": "f56e55503bebba456ce7a8bcc4bcab3a5c34fa71d57c1e0243179dd4ba429fbf"
|
|
27
33
|
}
|
|
28
34
|
],
|
|
29
35
|
"build": {
|
|
30
|
-
"asrSourceSha256": "
|
|
36
|
+
"asrSourceSha256": "8b977f8131ed154f517dab2eb7abd95fd2d4e31d22f526f544af79587fe88f5e",
|
|
31
37
|
"clangVersion": "Apple clang version 21.0.0 (clang-2100.1.1.101)",
|
|
32
|
-
"embeddingSourceSha256": "
|
|
38
|
+
"embeddingSourceSha256": "8fb22c36a2dc5dc849abd16040d839f170c12ee30ce85bf956227102def8a349",
|
|
39
|
+
"mediaSourceSha256": "288bfdea3eba7c21434ae4188b77dd75ab0aced83a662ade802890a59bcdf101",
|
|
40
|
+
"rustcVersion": "rustc 1.97.1 (8bab26f4f 2026-07-14)",
|
|
33
41
|
"swiftVersion": "Apple Swift version 6.3.3 (swiftlang-6.3.3.1.3 clang-2100.1.1.101)"
|
|
34
42
|
},
|
|
35
43
|
"minimumMacOS": "26.0",
|
|
36
|
-
"packageVersion": "0.
|
|
44
|
+
"packageVersion": "0.5.0",
|
|
37
45
|
"platform": "darwin",
|
|
38
46
|
"schemaVersion": 1,
|
|
39
|
-
"sourceCommit": "
|
|
47
|
+
"sourceCommit": "bbfe2485826242f7a314735a9a788ab3311096eb"
|
|
40
48
|
}
|
package/package.json
CHANGED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
/** Public Ghostget SDK bridge; its complete inherited group belongs to the caller. */
|
|
2
|
-
import { readFile } from 'node:fs/promises';
|
|
3
|
-
import { processIdentity } from '../worker/identity.js';
|
|
4
|
-
import { SpongeError } from '../errors.js';
|
|
5
|
-
try {
|
|
6
|
-
const config = JSON.parse(await readFile(process.argv[2], 'utf8'));
|
|
7
|
-
const sdk = await import(import.meta.resolve('@hraness/ghostget/media-acquisition', `${config.runtimeRoot}/package.json`));
|
|
8
|
-
const identity = await processIdentity(process.pid);
|
|
9
|
-
if (!identity)
|
|
10
|
-
throw new SpongeError('unavailable');
|
|
11
|
-
const result = await sdk.acquireEphemeralMedia({
|
|
12
|
-
...config.request,
|
|
13
|
-
processGroup: 'inherit',
|
|
14
|
-
supervisor: { requestId: config.requestId, processGroupId: process.pid, processStartIdentity: identity.started },
|
|
15
|
-
});
|
|
16
|
-
process.stdout.write(JSON.stringify(result));
|
|
17
|
-
}
|
|
18
|
-
catch {
|
|
19
|
-
process.stdout.write(JSON.stringify({ ok: false, code: 'unavailable', cleanup: 'settled' }));
|
|
20
|
-
process.exitCode = 1;
|
|
21
|
-
}
|
|
File without changes
|