@hraness/sponge 0.2.5 → 0.3.0

package/dist/src/importers/custody.js

@@ -2,33 +2,51 @@ import { randomUUID } from 'node:crypto';
 import { join } from 'node:path';
 import { rm } from 'node:fs/promises';
 import { acquireWorkerCustody, proveAttemptCleanup } from '../worker/custody.js';
-import { directory, writeJournal, readJournal, syncDir } from '../worker/journal.js';
+import { directory, writeJournal, readJournal } from '../worker/journal.js';
+import { syncDirectoryAsync } from '../util/fsync.js';
 import { supervisedRunner } from '../worker/supervisor.js';
-const pending = () => Object.assign(Error('cleanup-required'), { code: 'cleanup-required' });
+import { SpongeError, codeOf } from '../errors.js';
+const pending = () => new SpongeError('cleanup-required');
 /** Read-only verification still owns subprocesses, including during an import preview. */
 export async function withImportVerification(libraryRoot, verify) {
     const custody = await acquireWorkerCustody(libraryRoot), attemptId = randomUUID();
     let scratchRoot, unknown = false;
     try {
         const attempt = await directory(join(custody.directory, 'attempts', attemptId));
-        await writeJournal(join(attempt, 'import-verification.json'), { schemaVersion: 1, kind: 'import-verification', attemptId, ownerIncarnation: custody.incarnation });
+        await writeJournal(join(attempt, 'import-verification.json'), {
+            schemaVersion: 1,
+            kind: 'import-verification',
+            attemptId,
+            ownerIncarnation: custody.incarnation,
+        });
         scratchRoot = await directory(join(libraryRoot, 'staging', 'import-verification-' + attemptId));
-        await writeJournal(join(scratchRoot, 'ownership.json'), { schemaVersion: 1, kind: 'import-verification', requestId: attemptId, ownerIncarnation: custody.incarnation });
-        return await verify({ scratchRoot, runProcess: supervisedRunner(custody.directory, custody.incarnation, attemptId) });
+        await writeJournal(join(scratchRoot, 'ownership.json'), {
+            schemaVersion: 1,
+            kind: 'import-verification',
+            requestId: attemptId,
+            ownerIncarnation: custody.incarnation,
+        });
+        return await verify({
+            scratchRoot,
+            runProcess: supervisedRunner(custody.directory, custody.incarnation, attemptId),
+        });
     }
     catch (error) {
-        unknown = error?.code === 'cleanup-required' || (error instanceof Error && error.message === 'cleanup-required');
+        unknown = codeOf(error) === 'cleanup-required';
         throw error;
     }
     finally {
-        if (unknown || !await proveAttemptCleanup(custody.directory, custody.incarnation, attemptId))
+        if (unknown || !(await proveAttemptCleanup(custody.directory, custody.incarnation, attemptId)))
             throw pending();
         if (scratchRoot) {
             const marker = await readJournal(join(scratchRoot, 'ownership.json'));
-            if (marker.schemaVersion !== 1 || marker.kind !== 'import-verification' || marker.requestId !== attemptId || marker.ownerIncarnation !== custody.incarnation)
+            if (marker.schemaVersion !== 1 ||
+                marker.kind !== 'import-verification' ||
+                marker.requestId !== attemptId ||
+                marker.ownerIncarnation !== custody.incarnation)
                 throw pending();
             await rm(scratchRoot, { recursive: true });
-            await syncDir(join(libraryRoot, 'staging'));
+            await syncDirectoryAsync(join(libraryRoot, 'staging'));
         }
         await custody.release();
     }

package/dist/src/importers/inspect.js

@@ -3,84 +3,153 @@ import { join } from 'node:path';
 import { limits } from '../../contracts/index.js';
 import { validateAsset } from '../../contracts/validate.js';
 import { object, metadataFromMarkdown } from '../acquisition/normalize.js';
-import { readConfined, hash } from '../acquisition/files.js';
+import { readConfined } from '../acquisition/files.js';
+import { sha256 as hash } from '../util/hash.js';
 import { withImportVerification } from './custody.js';
 import { inspectCaptureRuntime } from '../acquisition/runtime.js';
 import { articleMetadata } from '../metadata/article.js';
 import { bundleBytes } from '../app/bundle-files.js';
+import { SpongeError } from '../errors.js';
 export function releasedBundleVerifier(runtime, libraryRoot, signal) {
     return async (root, kind) => {
         const checked = await inspectCaptureRuntime(runtime.root, runtime.pins), helper = fileURLToPath(new URL(import.meta.url.endsWith('.ts') ? './verify-helper.ts' : './verify-helper.js', import.meta.url));
         return withImportVerification(libraryRoot, async ({ scratchRoot, runProcess }) => {
             const args = [helper, checked.root, root, kind];
-            const result = await runProcess(process.platform === 'darwin' ? '/usr/bin/sandbox-exec' : checked.bun, process.platform === 'darwin' ? ['-p', '(version 1) (allow default) (deny network*)', checked.bun, ...args] : args, { env: { PATH: '/usr/bin:/bin', HOME: checked.root, TMPDIR: scratchRoot, LANG: 'en_US.UTF-8' }, deadlineMs: 45000, maxOutputBytes: 65536, signal });
+            const result = await runProcess(process.platform === 'darwin' ? '/usr/bin/sandbox-exec' : checked.bun, process.platform === 'darwin'
+                ? ['-p', '(version 1) (allow default) (deny network*)', checked.bun, ...args]
+                : args, {
+                env: { PATH: '/usr/bin:/bin', HOME: checked.root, TMPDIR: scratchRoot, LANG: 'en_US.UTF-8' },
+                deadlineMs: 45000,
+                maxOutputBytes: 65536,
+                signal,
+            });
             if (result.cleanup !== 'settled')
-                throw Error('cleanup-required');
+                throw new SpongeError('cleanup-required');
             if (result.stopped)
-                throw Error(result.stopped);
+                throw new SpongeError(result.stopped);
             if (result.code !== 0)
-                throw Error('unsupported');
+                throw new SpongeError('unsupported');
             return JSON.parse(result.stdout);
         });
     };
 }
-function imageType(bytes) { if (bytes[0] === 255 && bytes[1] === 216)
-    return 'image/jpeg'; if (bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])))
-    return 'image/png'; if (bytes.toString('ascii', 0, 4) === 'RIFF' && bytes.toString('ascii', 8, 12) === 'WEBP')
-    return 'image/webp'; if (/^GIF8[79]a/.test(bytes.toString('ascii', 0, 6)))
-    return 'image/gif'; if (bytes.toString('ascii', 4, 12) === 'ftypavif')
-    return 'image/avif'; throw Error('unsupported'); }
+function imageType(bytes) {
+    if (bytes[0] === 255 && bytes[1] === 216)
+        return 'image/jpeg';
+    if (bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])))
+        return 'image/png';
+    if (bytes.toString('ascii', 0, 4) === 'RIFF' && bytes.toString('ascii', 8, 12) === 'WEBP')
+        return 'image/webp';
+    if (/^GIF8[79]a/.test(bytes.toString('ascii', 0, 6)))
+        return 'image/gif';
+    if (bytes.toString('ascii', 4, 12) === 'ftypavif')
+        return 'image/avif';
+    throw new SpongeError('unsupported');
+}
 /** An import-time digest never upgrades a missing historical document digest. */
 export async function inspectImportBundle(root, verify) {
+    // A bundle and its verifier report are untrusted: malformed JSON, a listed file that is missing, a bad URL or asset record is an integrity failure.
+    try {
+        return await inspectBundle(root, verify);
+    }
+    catch (error) {
+        if (error instanceof SpongeError)
+            throw error;
+        throw new SpongeError('integrity', { cause: error });
+    }
+}
+async function inspectBundle(root, verify) {
     const manifestBytes = await readConfined(root, 'capture.json', limits.metadataBytes), manifest = object(JSON.parse(manifestBytes.toString('utf8'))), kind = manifest.kind === 'pdf' ? 'pdf' : 'article';
     if (kind === 'pdf' ? manifest.schemaVersion !== 1 : ![1, 2, 3, 4].includes(Number(manifest.schemaVersion)))
-        throw Error('unsupported');
+        throw new SpongeError('unsupported');
     const result = object(await verify(root, kind)), inspection = object(result.inspection), document = object(kind === 'pdf' ? inspection.markdown : inspection.document);
-    if (inspection.schemaVersion !== manifest.schemaVersion || !['verified', 'unavailable'].includes(String(document.integrity)) || !Array.isArray(inspection.assets) || inspection.assets.length > limits.imagesPerItem)
-        throw Error('integrity');
-    if (result.ok !== true && (!Array.isArray(result.issues) || !result.issues.length || result.issues.some(raw => { const issue = object(raw); return !(issue.kind === 'document-integrity' && issue.path === document.path && document.integrity === 'unavailable' && (kind === 'article' || issue.integrity === 'unavailable')); })))
-        throw Error('integrity');
-    if (inspection.status !== undefined && inspection.status !== null && !['complete', 'partial'].includes(String(inspection.status)))
-        throw Error('unsupported');
-    const markdownBytes = await readConfined(root, String(document.path), limits.markdownBytes), markdown = { relativePath: String(document.path), bytes: markdownBytes.length, sha256: hash(markdownBytes), mediaType: 'text/markdown' };
+    if (inspection.schemaVersion !== manifest.schemaVersion ||
+        !['verified', 'unavailable'].includes(String(document.integrity)) ||
+        !Array.isArray(inspection.assets) ||
+        inspection.assets.length > limits.imagesPerItem)
+        throw new SpongeError('integrity');
+    if (result.ok !== true &&
+        (!Array.isArray(result.issues) ||
+            !result.issues.length ||
+            result.issues.some((raw) => {
+                const issue = object(raw);
+                return !(issue.kind === 'document-integrity' &&
+                    issue.path === document.path &&
+                    document.integrity === 'unavailable' &&
+                    (kind === 'article' || issue.integrity === 'unavailable'));
+            })))
+        throw new SpongeError('integrity');
+    if (inspection.status !== undefined &&
+        inspection.status !== null &&
+        !['complete', 'partial'].includes(String(inspection.status)))
+        throw new SpongeError('unsupported');
+    const markdownBytes = await readConfined(root, String(document.path), limits.markdownBytes), markdown = {
+        relativePath: String(document.path),
+        bytes: markdownBytes.length,
+        sha256: hash(markdownBytes),
+        mediaType: 'text/markdown',
+    };
     validateAsset(markdown);
     if (markdown.sha256 !== document.sha256 || markdown.bytes !== document.bytes)
-        throw Error('integrity');
+        throw new SpongeError('integrity');
     let unavailable = document.integrity === 'unavailable';
     const assets = [];
     let total = 0;
     for (const raw of inspection.assets) {
         const asset = object(raw);
         if (!['verified', 'unavailable'].includes(String(asset.integrity)))
-            throw Error('integrity');
+            throw new SpongeError('integrity');
         unavailable ||= asset.integrity === 'unavailable';
         const bytes = await readConfined(root, String(asset.path), limits.imageBytes);
         total += bytes.length;
         if (total > 500 * 1024 ** 2)
-            throw Error('limit-exceeded');
-        const value = { relativePath: String(asset.path), bytes: bytes.length, sha256: hash(bytes), mediaType: imageType(bytes), relationship: 'image' };
+            throw new SpongeError('limit-exceeded');
+        const value = {
+            relativePath: String(asset.path),
+            bytes: bytes.length,
+            sha256: hash(bytes),
+            mediaType: imageType(bytes),
+            relationship: 'image',
+        };
         validateAsset(value);
         const expectedHash = kind === 'pdf' ? asset.sha256 : asset.expectedSha256, expectedBytes = kind === 'pdf' ? asset.bytes : asset.expectedBytes;
         if (asset.integrity === 'verified' && (expectedHash !== value.sha256 || expectedBytes !== value.bytes))
-            throw Error('integrity');
+            throw new SpongeError('integrity');
         assets.push(value);
     }
     if (kind === 'pdf') {
         const source = object(inspection.source);
         if (source.integrity !== 'verified')
-            throw Error('integrity');
-        const asset = { relativePath: String(source.path), sha256: String(source.sha256), bytes: Number(source.bytes), mediaType: 'application/pdf', relationship: 'source-pdf' };
+            throw new SpongeError('integrity');
+        const asset = {
+            relativePath: String(source.path),
+            sha256: String(source.sha256),
+            bytes: Number(source.bytes),
+            mediaType: 'application/pdf',
+            relationship: 'source-pdf',
+        };
         validateAsset(asset);
-        for await (const _ of bundleBytes(root, asset)) { }
+        for await (const _ of bundleBytes(root, asset)) {
+        }
         assets.push(asset);
     }
-    if (assets.length !== new Set(assets.map(a => a.relativePath)).size)
-        throw Error('integrity');
-    const sourceUrl = kind === 'pdf' ? (object(manifest.source).requestedUrl ?? pathToFileURL(join(root, String(object(inspection.source).path))).href) : inspection.sourceUrl;
-    if (typeof sourceUrl !== 'string' || sourceUrl.length > 16384 || !['http:', 'https:', 'file:'].includes(new URL(sourceUrl).protocol))
-        throw Error('integrity');
-    const observedAt = typeof manifest.capturedAt === 'string' && Number.isFinite(Date.parse(manifest.capturedAt)) ? manifest.capturedAt : '1970-01-01T00:00:00.000Z';
-    const metadata = kind === 'article' ? (manifest.metadata === undefined ? metadataFromMarkdown(markdownBytes.toString('utf8'), observedAt) : articleMetadata(manifest.metadata, observedAt)) : [];
+    if (assets.length !== new Set(assets.map((a) => a.relativePath)).size)
+        throw new SpongeError('integrity');
+    const sourceUrl = kind === 'pdf'
+        ? (object(manifest.source).requestedUrl ?? pathToFileURL(join(root, String(object(inspection.source).path))).href)
+        : inspection.sourceUrl;
+    if (typeof sourceUrl !== 'string' ||
+        sourceUrl.length > 16384 ||
+        !['http:', 'https:', 'file:'].includes(new URL(sourceUrl).protocol))
+        throw new SpongeError('integrity');
+    const observedAt = typeof manifest.capturedAt === 'string' && Number.isFinite(Date.parse(manifest.capturedAt))
+        ? manifest.capturedAt
+        : '1970-01-01T00:00:00.000Z';
+    const metadata = kind === 'article'
+        ? manifest.metadata === undefined
+            ? metadataFromMarkdown(markdownBytes.toString('utf8'), observedAt)
+            : articleMetadata(manifest.metadata, observedAt)
+        : [];
     if (kind === 'pdf') {
         const document = object(manifest.document ?? {});
         for (const field of ['title', 'author', 'subject', 'creator', 'producer', 'createdAt', 'modifiedAt']) {
@@ -89,9 +158,27 @@ export async function inspectImportBundle(root, verify) {
                 metadata.push({ field, value, source: 'pdf', observedAt, rule: 'wordcell-pdf-v1', selected: true });
         }
     }
-    metadata.push({ field: 'historicalIntegrity', value: unavailable ? 'unavailable' : 'verified', source: kind === 'pdf' ? 'pdf' : 'html', observedAt, rule: 'sponge-import-v1' });
+    metadata.push({
+        field: 'historicalIntegrity',
+        value: unavailable ? 'unavailable' : 'verified',
+        source: kind === 'pdf' ? 'pdf' : 'html',
+        observedAt,
+        rule: 'sponge-import-v1',
+    });
     if (hash(await readConfined(root, 'capture.json', limits.metadataBytes)) !== hash(manifestBytes))
-        throw Error('integrity');
-    const manifestSha256 = hash(manifestBytes), importHash = hash(JSON.stringify([manifestSha256, markdown.sha256, ...assets.map(a => [a.relativePath, a.sha256, a.bytes])]));
-    return { root, kind, sourceUrl, schemaVersion: Number(manifest.schemaVersion), status: unavailable || inspection.status !== 'complete' ? 'partial' : 'complete', markdown, assets, metadata, manifestSha256, historicalIntegrity: unavailable ? 'unavailable' : 'verified', importHash };
+        throw new SpongeError('integrity');
+    const manifestSha256 = hash(manifestBytes), importHash = hash(JSON.stringify([manifestSha256, markdown.sha256, ...assets.map((a) => [a.relativePath, a.sha256, a.bytes])]));
+    return {
+        root,
+        kind,
+        sourceUrl,
+        schemaVersion: Number(manifest.schemaVersion),
+        status: unavailable || inspection.status !== 'complete' ? 'partial' : 'complete',
+        markdown,
+        assets,
+        metadata,
+        manifestSha256,
+        historicalIntegrity: unavailable ? 'unavailable' : 'verified',
+        importHash,
+    };
 }

package/dist/src/importers/jungle-digest.d.ts

@@ -1,18 +1,15 @@
-import { type DigestIssue, type DigestV1 } from '../digest/schema.js';
+import { type DigestIssue, type DigestV2 } from '../digest/schema.js';
 import { type Form, type QuoteMatch } from '../digest/pipeline.js';
 /**
- * Pure conversion of one reviewed Jungle Reading digest note (`kb/notes/reading/<slug>.md`) into Digest v1. Jungle keeps
- * quotes in their own section, so each quote is assigned to the idea it shares the most content words with.
+ * Pure conversion of one reviewed Jungle Reading digest note (`kb/notes/reading/<slug>.md`) into Digest v2. Jungle keeps
+ * quotes in their own section, so each verbatim quote is assigned to the idea it shares the most content words with.
+ * Owner text is never cut or rewritten: a note that does not fit v2 as written is reported, not converted.
  * Nothing here reads files; callers pass the note and, when available, its capture Markdown.
  */
 export declare const jungleTags: readonly ["agent-eng", "ai", "work", "business", "creative-coding", "finance", "music", "internet", "security", "history", "art", "hardware", "climate", "article", "interactive", "code", "x", "video"];
-/** Tags that name how a source is consumed; Digest v1 carries that as `form`. */
+/** Tags that name how a source is consumed; a digest carries that as `form`. */
 export declare const jungleFormatTags: readonly ["article", "interactive", "x", "video"];
-export declare class JungleDigestError extends Error {
-    readonly code: 'not-digest' | 'malformed' | 'no-capture';
-    constructor(code: 'not-digest' | 'malformed' | 'no-capture');
-}
-export interface JungleQuote {
+interface JungleQuote {
     text: string;
     attribution: string;
     speaker: string;
@@ -32,25 +29,27 @@ export interface JungleNote {
     }[];
     quotes: JungleQuote[];
 }
-export type Lossy = 'context-trimmed' | 'quote-unassigned' | 'quote-missing' | 'idea-unquoted' | 'no-content-tag';
+/**
+ * Why a Jungle quote is left out of the digest: not found as written in the capture (`snapped` matches differ in markup),
+ * under four words, over the quote budget (count, words or source share), or every idea already holds two.
+ */
+export type QuoteDrop = 'not-verbatim' | 'too-short' | 'budget' | 'overflow';
 export interface JungleConversion {
     note: JungleNote;
-    /** Every idea, including unquoted ones: the evaluation gold. */
-    gold: DigestV1;
-    /** Per Jungle quote: the gold idea it was assigned to, and how it located in the capture. */
+    /** The note as Digest v2: the Gist's first sentence as lead and the rest as context, every idea, and the quotes that survive. */
+    gold: DigestV2;
+    /** Per Jungle quote, in note order: the idea it went to (null when dropped), how it located in the capture, and why it was dropped. */
     quotes: {
         idea: number | null;
         match?: QuoteMatch;
+        dropped?: QuoteDrop;
     }[];
-    lossy: Lossy[];
-    /** checkDigest issues of the gold against the capture; empty without a capture. */
+    /** Why the gold is not a valid Digest v2: shape issues (an empty detail), then checkDigest issues against the capture when one is given. */
     issues: DigestIssue[];
-    /** The gold with unlocated quotes and then unquoted ideas dropped, when that still passes every v1 rule. */
-    importable?: DigestV1;
-    /** Gold passes every v1 rule against the capture with nothing dropped or trimmed. `kind` is never gold: Jungle has no kinds. */
-    clean: boolean;
+    /** The gold, when a capture was given and the gold passes every v2 rule against it. */
+    digest?: DigestV2;
 }
-export type Yaml = string | boolean | Yaml[] | {
+type Yaml = string | boolean | Yaml[] | {
     [k: string]: Yaml;
 };
 /** Capture Markdown without its YAML frontmatter: what a Sponge revision of the same page would hold. */
@@ -58,7 +57,7 @@ export declare function captureBody(capture: string): string;
 export declare function parseJungleNote(markdown: string): JungleNote;
 /**
  * One Jungle Reading note of any form (digest, social, note, conversation), kept whole: the exact Markdown and its hash,
- * the parsed frontmatter, and the gist, ideas and quotes where the form has them. Nothing is trimmed to Digest v1 limits.
+ * the parsed frontmatter, and the gist, ideas and quotes where the form has them. Nothing is trimmed to digest limits.
  * `capture` is the evidence the note is about: the first `synthesizes` relation, else the first `evidenced-by`.
  */
 export interface JungleEditorial {
@@ -84,14 +83,22 @@ export declare function parseJungleEditorial(markdown: string): JungleEditorial;
 /** Content-word stems: lowercase, stopwords out, a crude plural/tense strip. Shared by idea assignment and lexical coverage. */
 export declare function contentTerms(text: string): Set<string>;
 export declare function termSimilarity(a: Set<string>, b: Set<string>): number;
-/** Best-first assignment: every idea gets at most one quote before any gets a second; zero-overlap pairs are never made. */
+/**
+ * Each quote in order goes to the idea sharing the most content words with its claim and detail (ties: the earliest idea),
+ * else to the next best idea with room; null when every idea already holds `cap` quotes.
+ */
 export declare function assignQuotes(ideas: {
     claim: string;
     detail: string;
 }[], quotes: {
     text: string;
-}[]): (number | null)[];
+}[], cap?: number): (number | null)[];
 export declare const firstSentence: (text: string) => readonly [string, string];
 export declare function formOf(note: JungleNote): Form;
-/** Convert one Jungle digest note. With `capture` (the capture Markdown, frontmatter allowed), quotes are located and the gold is checked. */
+/**
+ * Convert one Jungle digest note. With `capture` (the capture Markdown, frontmatter allowed), each quote is kept only when it
+ * occurs as written (folded typography only) and has at least four words; whole quotes then leave, longest and then latest in
+ * the note first, until the rest fit the quote budget (never the last one); survivors are assigned to ideas, and the gold is checked.
+ */
 export declare function convertJungleDigest(noteMarkdown: string, capture?: string): JungleConversion;
+export {};