@hraness/sponge 0.4.1 → 0.5.1

package/README.md

@@ -1,6 +1,6 @@
 # Sponge
 
-Sponge is a local library for articles, images, PDFs and media transcripts. Capturing runs on your machine through Ghostget; saved
+Sponge is a local library for articles, images, PDFs and media transcripts. Capturing runs on your machine through Wordcell and Sponge's media helper; saved
 content stays in a private SQLite database and adjacent hashed asset files.
 
 Version 0.2.0 introduces the local-library CLI; earlier 0.1.x commands are inactive.
@@ -34,8 +34,9 @@ revision and preserves the old one if capture fails. `--json` provides structure
 output. Errors are categorical JSON on stderr; exit codes are 0 success,
 1 failure, 2 invalid input, 3 partial content, 4 action required, and 130 cancelled.
 
-Capture setup explicitly downloads pinned Ghostget, Wordcell and Bun artifacts
-plus frozen dependencies. Read/search never download runtimes or contact source
+Capture setup explicitly downloads pinned Wordcell and Bun artifacts plus frozen
+dependencies, and on macOS 26 on Apple Silicon Sponge's media helper from the
+Sponge release. Read/search never download runtimes or contact source
 websites. Setup reports disk requirements, a monitored 2 GiB temporary installation
 limit, a 1 GiB retained runtime limit and a 30-minute dependency installation
 deadline. Temporary dependency caches are removed
@@ -59,8 +60,8 @@ node bin/sponge.js add https://example.com/episode --kind media --wait
 ```
 
 PDF capture requires Poppler (`pdfinfo` and `pdftohtml`). Tesseract enables
-optional local OCR. Provider media acquisition requires yt-dlp; audio validation
-requires ffprobe from FFmpeg. Install these separately, for example with
+optional local OCR. Media capture runs Sponge's native media helper, which uses
+yt-dlp and FFmpeg (ffprobe) from your PATH or Homebrew. Install these separately, for example with
 `brew install poppler tesseract yt-dlp ffmpeg` on macOS. Capture setup does not
 install system tools.
 
@@ -86,6 +87,7 @@ that changes them reports `runtime-missing` until setup runs again.
 ```sh
 sponge setup --asr
 sponge add https://example.com/episode --kind media --wait --language en
+sponge add https://youtu.be/VIDEO_ID --wait   # YouTube, Vimeo, SoundCloud and audio or video files are media without --kind
 sponge refresh ITEM --wait --language en --force-local
 sponge setup --embedding
 sponge index create
@@ -204,21 +206,20 @@ HTTPS. See [the decision record](docs/decisions/publish-feed.md).
 ### Selected authenticated connections
 
 The initial development interface accepts private `connections.json` in the
-library (mode `0600`). It contains references to existing Ghostget accounts,
-never cookies or tokens. A future setup phase adds connection-management commands.
+library (mode `0600`). It names a signed-in browser you attached over the Chrome
+DevTools Protocol on loopback, never cookies or tokens.
 
 ```json
-{"schemaVersion":1,"connections":[{"id":"personal","kind":"account","stateHome":"/absolute/ghostget-auth","accountId":"personal"}]}
+{"schemaVersion":1,"connections":[{"id":"personal","kind":"cdp","endpoint":"http://127.0.0.1:9222"}]}
 ```
 
-Use `add URL --connection personal --wait`. A public capture never silently
-selects your browser or account. For articles, attached browser connections use
-`kind: "cdp"` and an explicitly selected local `endpoint`; attached browsers remain
-open. PDF and media capture support selected Ghostget accounts, not CDP. Remote
-PDF acquisition requires public network addresses; `--allow-private-network` is
-not supported for PDF capture. Live authenticated remote-PDF qualification remains
-pending; failures never fall back to a different account or a public request.
-Changing connection identity creates a distinct private source realm.
+Use `add URL --connection personal --wait` for an article. A public capture never
+silently selects your browser, and an attached browser stays open. PDF and media
+capture are public only. Remote PDF acquisition requires public network addresses;
+`--allow-private-network` is not supported for PDF capture. Saved GhostGet account
+connections from earlier versions are retired: their items keep their realm, and a
+capture through one fails with reason `connection-retired`. Changing connection
+identity creates a distinct private source realm.
 
 ## Design and validation
 

package/dist/src/acquisition/capture.d.ts

@@ -1,11 +1,8 @@
 import type { CaptureResult, ErrorEnvelope } from '../../contracts/index.js';
 import { runOwned } from './process.js';
 import { type CaptureRuntime } from './runtime.js';
+/** A signed-in browser the user attached over CDP on loopback. */
 export type CaptureConnection = {
-    kind: 'account';
-    stateHome: string;
-    accountId: string;
-} | {
     kind: 'cdp';
     endpoint: string;
 };

package/dist/src/acquisition/capture.js

@@ -6,7 +6,7 @@ import { privateRoot, physicalBytes } from './files.js';
 import { runOwned, isolatedEnvironment } from './process.js';
 import { inspectCaptureRuntime } from './runtime.js';
 import { settleCaptureRoot } from './cleanup.js';
-import { failure, upstreamFailure } from './errors.js';
+import { failure, clipFailure } from './errors.js';
 import { normalizeBundle, object } from './normalize.js';
 import { SpongeError, publicCode, codeOf } from '../errors.js';
 export async function captureArticle(request) {
@@ -44,9 +44,10 @@ export async function captureArticle(request) {
     const deadline = request.deadlineMs ?? limits.captureDeadlineMs;
     if (!Number.isSafeInteger(deadline) || deadline < 100 || deadline > limits.captureDeadlineMs)
         return failure(requestId, 'invalid-input');
+    const clipHelper = fileURLToPath(new URL(import.meta.url.endsWith('.ts') ? './clip-helper.ts' : './clip-helper.js', import.meta.url));
     const args = [
-        runtime.ghostget,
-        'clip',
+        clipHelper,
+        runtime.root,
         request.sourceUrl,
         'article',
         '--media',
@@ -64,28 +65,18 @@ export async function captureArticle(request) {
     ];
     if (request.allowPrivateNetwork)
         args.push('--allow-private-network');
-    let selectedState;
-    if (request.connection?.kind === 'account') {
-        if (!/^[a-zA-Z0-9_.-]{1,128}$/.test(request.connection.accountId))
-            return failure(requestId, 'invalid-input');
-        try {
-            selectedState = await privateRoot(request.connection.stateHome);
-        }
-        catch {
-            return failure(requestId, 'repair-auth');
-        }
-        args.push('--auth', request.connection.accountId, '--mode', 'http');
-    }
-    else if (request.connection?.kind === 'cdp') {
+    if (request.connection) {
         if (!/^(?:[1-9][0-9]{0,4}|(?:http|ws):\/\/127\.0\.0\.1:[1-9][0-9]{0,4}(?:\/[^\s]*)?)$/.test(request.connection.endpoint))
             return failure(requestId, 'invalid-input');
-        args.push('--cdp', request.connection.endpoint);
+        // Wordcell takes the loopback DevTools port alone.
+        const endpoint = request.connection.endpoint;
+        args.push('--cdp', /^[0-9]+$/.test(endpoint) ? endpoint : new URL(endpoint).port);
     }
     else
         args.push('--mode', 'http');
     const root = await realpath(await mkdtemp(join(base, 'capture-'))), home = join(root, 'home'), output = join(root, 'bundles');
     args.push('--output', output);
-    const env = isolatedEnvironment(home, home, selectedState ?? join(home, 'state'));
+    const env = isolatedEnvironment(home, home);
     const controller = new AbortController();
     const abort = () => controller.abort();
     signal?.addEventListener('abort', abort, { once: true });
@@ -127,7 +118,7 @@ export async function captureArticle(request) {
             graceMs: 5000,
         });
         cleanup = result.cleanup;
-        // Ghostget reports escaped browser custody separately from generic descendants.
+        // An attached browser can start helpers outside the owned process group.
         let summary;
         try {
             summary = JSON.parse(result.stdout);
@@ -135,10 +126,8 @@ export async function captureArticle(request) {
         catch {
             summary = null;
         }
-        const upstream = upstreamFailure(summary, requestId);
-        if (upstream.code === 'cleanup-required')
-            cleanup = 'pending';
-        if (cleanup === 'settled' && request.connection?.kind === 'cdp') {
+        const upstream = clipFailure(summary, requestId);
+        if (cleanup === 'settled' && request.connection) {
             cleanup = 'pending';
             const census = await (request.runProcess ?? runOwned)('/bin/ps', ['-axo', 'command'], {
                 env,

package/dist/src/acquisition/clip-helper.js

@@ -0,0 +1,14 @@
+// Executed only by the pinned Bun runtime. The import is Wordcell's public clip entry point.
+const runtime = process.argv[2];
+try {
+    const { main } = await import(import.meta.resolve('@hraness/wordcell/clip/cli', `${runtime}/package.json`));
+    process.exitCode = await main(process.argv.slice(3), process.env, {
+        stdout: (value) => process.stdout.write(value),
+        stderr: (value) => process.stderr.write(value),
+    });
+}
+catch {
+    process.stdout.write(JSON.stringify({ ok: false }));
+    process.exitCode = 1;
+}
+export {};

package/dist/src/acquisition/errors.d.ts

@@ -1,3 +1,4 @@
 import type { ErrorCode, ErrorEnvelope } from '../../contracts/index.js';
 export declare const failure: (requestId: string, code: ErrorCode, phase?: ErrorEnvelope["phase"], retryable?: boolean) => ErrorEnvelope;
-export declare function upstreamFailure(value: unknown, requestId: string): ErrorEnvelope;
+/** Wordcell's clip answers with a JSON summary; prose is never classified. A summary that is not success means the source gave no usable capture. */
+export declare function clipFailure(value: unknown, requestId: string): ErrorEnvelope;

package/dist/src/acquisition/errors.js

@@ -1,35 +1,6 @@
-const mappings = {
-    unknown: 'unclassified',
-    'permission-required': 'grant-permission',
-    'auth-required': 'repair-auth',
-    'invalid-input': 'invalid-input',
-    cancelled: 'cancelled',
-    timeout: 'unavailable',
-    transient: 'unavailable',
-    unsupported: 'unsupported',
-    'cleanup-pending': 'cleanup-required',
-    'rate-limited': 'rate-limited',
-};
 export const failure = (requestId, code, phase = 'acquisition', retryable = false) => ({ schemaVersion: 1, requestId, ok: false, code, phase, retryable });
-export function upstreamFailure(value, requestId) {
-    const fallback = failure(requestId, 'unclassified');
-    if (!value || typeof value !== 'object' || Array.isArray(value))
-        return fallback;
-    const outer = value;
-    if (outer.ok !== false)
-        return fallback;
-    const raw = outer.captureError;
-    if (!raw || typeof raw !== 'object' || Array.isArray(raw))
-        return fallback;
-    const typed = raw;
-    if (Object.keys(typed).some((k) => !['schemaVersion', 'code', 'retryable'].includes(k)) ||
-        typed.schemaVersion !== 1 ||
-        typeof typed.code !== 'string' ||
-        !Object.hasOwn(mappings, typed.code) ||
-        typeof typed.retryable !== 'boolean')
-        return fallback;
-    const code = mappings[typed.code];
-    if (typed.retryable && !['timeout', 'transient', 'rate-limited'].includes(typed.code))
-        return fallback;
-    return failure(requestId, code, code === 'cleanup-required' ? 'cleanup' : 'acquisition', typed.retryable);
+/** Wordcell's clip answers with a JSON summary; prose is never classified. A summary that is not success means the source gave no usable capture. */
+export function clipFailure(value, requestId) {
+    const answered = !!value && typeof value === 'object' && !Array.isArray(value) && 'ok' in value;
+    return failure(requestId, answered ? 'unavailable' : 'unclassified');
 }