@hraness/sponge 0.2.3 → 0.2.5

package/dist/src/importers/jungle-digest.js

@@ -0,0 +1,221 @@
+import { createHash } from 'node:crypto';
+import { foldText, locateAll, words } from '../annotations/quote.js';
+import { checkDigest, digestLimits as L } from '../digest/schema.js';
+import { minQuoteWords, snapQuote } from '../digest/pipeline.js';
+/**
+ * Pure conversion of one reviewed Jungle Reading digest note (`kb/notes/reading/<slug>.md`) into Digest v1. Jungle keeps
+ * quotes in their own section, so each quote is assigned to the idea it shares the most content words with.
+ * Nothing here reads files; callers pass the note and, when available, its capture Markdown.
+ */
+export const jungleTags = ['agent-eng', 'ai', 'work', 'business', 'creative-coding', 'finance', 'music', 'internet', 'security', 'history', 'art', 'hardware', 'climate', 'article', 'interactive', 'code', 'x', 'video'];
+/** Tags that name how a source is consumed; Digest v1 carries that as `form`. */
+export const jungleFormatTags = ['article', 'interactive', 'x', 'video'];
+export class JungleDigestError extends Error {
+    code;
+    constructor(code) {
+        super(code);
+        this.code = code;
+        this.name = 'JungleDigestError';
+    }
+}
+const fail = (code = 'malformed') => { throw new JungleDigestError(code); };
+/** The frontmatter subset Jungle writes: block maps and lists of scalars, quoted or plain. */
+function parseYaml(lines) {
+    let i = 0;
+    const indent = (l) => l.length - l.trimStart().length;
+    const scalar = (v) => { v = v.trim(); if (v.startsWith('"')) {
+        try {
+            return JSON.parse(v);
+        }
+        catch {
+            return fail();
+        }
+    } if (v.startsWith("'"))
+        return v.endsWith("'") && v.length > 1 ? v.slice(1, -1).replace(/''/g, "'") : fail(); return v === 'true' ? true : v === 'false' ? false : v; };
+    const block = (at) => {
+        if (lines[i].trimStart().startsWith('- ')) {
+            const list = [];
+            while (i < lines.length && indent(lines[i]) === at && lines[i].trimStart().startsWith('- ')) {
+                const rest = lines[i].trimStart().slice(2);
+                if (/^[\w-]+:(\s|$)/.test(rest)) {
+                    lines[i] = ' '.repeat(at + 2) + rest;
+                    list.push(map(at + 2));
+                }
+                else {
+                    list.push(scalar(rest));
+                    i++;
+                }
+            }
+            return list;
+        }
+        return map(at);
+    };
+    const map = (at) => {
+        const out = {};
+        while (i < lines.length && indent(lines[i]) === at && !lines[i].trimStart().startsWith('- ')) {
+            const m = /^([\w-]+):(?:\s+(.*))?$/.exec(lines[i].trim()) ?? fail();
+            i++;
+            if (m[2] !== undefined && m[2] !== '')
+                out[m[1]] = scalar(m[2]);
+            else if (i < lines.length && (indent(lines[i]) > at || indent(lines[i]) === at && lines[i].trimStart().startsWith('- ')))
+                out[m[1]] = block(indent(lines[i]));
+            else
+                out[m[1]] = '';
+        }
+        return out;
+    };
+    const value = lines.length ? map(0) : {};
+    return i === lines.length ? value : fail();
+}
+const object = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
+const strings = (v) => Array.isArray(v) && v.every(x => typeof x === 'string') ? v : fail();
+/** Split a Markdown file into its frontmatter lines and body. */
+function frontmatter(markdown) {
+    const m = /^---\n([^]*?)\n---(?:\n|$)/.exec(markdown.replace(/\r\n/g, '\n'));
+    return m ? { head: m[1].split('\n').filter(l => l.trim() && !l.trimStart().startsWith('#')), body: markdown.replace(/\r\n/g, '\n').slice(m[0].length) } : { head: [], body: markdown };
+}
+/** Capture Markdown without its YAML frontmatter: what a Sponge revision of the same page would hold. */
+export function captureBody(capture) { return frontmatter(capture).body; }
+/** `## Heading` sections of a note body, trimmed. */
+function sectionsOf(body) {
+    const sections = new Map();
+    for (const part of body.split(/^## /m).slice(1)) {
+        const nl = part.indexOf('\n');
+        sections.set(part.slice(0, nl < 0 ? undefined : nl).trim(), nl < 0 ? '' : part.slice(nl + 1).trim());
+    }
+    return sections;
+}
+/** `- **Claim.** detail` bullets: Jungle's Ideas, and a conversation's Debate. */
+const ideaList = (text) => text.split('\n').filter(l => l.startsWith('- ')).map(l => { const m = /^- \*\*(.+?)\*\*\s*(.*)$/.exec(l.trim()) ?? fail(); return { claim: m[1].trim(), detail: m[2].trim() }; });
+/** Blockquotes ending `> Attribution: Speaker, context`. */
+const quoteList = (text) => text.split(/\n\s*\n/).filter(b => b.trim()).map(b => {
+    const lines = b.split('\n').map(l => /^>\s?(.*)$/.exec(l)?.[1] ?? fail()), at = lines.findIndex(l => l.startsWith('Attribution:'));
+    if (at < 1)
+        fail();
+    const text = lines.slice(0, at).join(' ').replace(/\s+/g, ' ').trim(), attribution = lines[at].slice('Attribution:'.length).trim();
+    return { text, attribution, speaker: attribution.split(/,|;|\s\(|\s—\s|\s-\s/)[0].trim() };
+});
+export function parseJungleNote(markdown) {
+    if (typeof markdown !== 'string' || markdown.length > 4 * 1024 * 1024)
+        fail();
+    const { head, body } = frontmatter(markdown), meta = parseYaml(head);
+    if (!object(meta) || !object(meta.reading))
+        throw new JungleDigestError('not-digest');
+    const reading = meta.reading;
+    if (reading.form !== 'digest')
+        fail('not-digest');
+    const sections = sectionsOf(body);
+    const gist = (sections.get('Gist') ?? fail()).replace(/\s+/g, ' ').trim();
+    const ideas = ideaList(sections.get('Ideas') ?? fail());
+    const quotes = quoteList(sections.get('Quotes') ?? '');
+    if (!gist || !ideas.length || quotes.some(q => !q.text || !q.speaker))
+        fail();
+    const authors = Array.isArray(reading.authors) ? reading.authors.map(a => object(a) && typeof a.name === 'string' ? a.name : fail()) : [];
+    const relations = object(meta.relations) ? meta.relations : {}, synth = Array.isArray(relations.synthesizes) ? strings(relations.synthesizes) : [];
+    return { slug: typeof reading.slug === 'string' ? reading.slug : fail(), title: typeof meta.title === 'string' ? meta.title : '', status: typeof reading.status === 'string' ? reading.status : '', sourceUrl: typeof reading.source_url === 'string' ? reading.source_url : '', authors, tags: meta.tags === undefined ? [] : strings(meta.tags), ...(synth[0] ? { capture: synth[0] } : {}), gist, ideas, quotes };
+}
+export function parseJungleEditorial(markdown) {
+    if (typeof markdown !== 'string' || markdown.length > 4 * 1024 * 1024)
+        fail();
+    const { head, body } = frontmatter(markdown), meta = parseYaml(head);
+    if (!object(meta) || !object(meta.reading))
+        throw new JungleDigestError('not-digest');
+    const reading = meta.reading, relations = object(meta.relations) ? meta.relations : {};
+    const form = reading.form, slug = reading.slug;
+    if (typeof form !== 'string' || !form || typeof slug !== 'string' || !/^[a-z0-9][a-z0-9-]{0,199}$/.test(slug))
+        return fail();
+    const capture = [relations.synthesizes, relations['evidenced-by']].flatMap(v => Array.isArray(v) ? strings(v) : [])[0] ?? fail('no-capture');
+    // A digest must parse strictly; the other forms keep whatever structure they have, and the Markdown keeps the rest.
+    const sections = sectionsOf(body), loose = (f) => { if (form === 'digest')
+        return f(); try {
+        return f();
+    }
+    catch {
+        return undefined;
+    } };
+    const gistText = sections.get('Gist') ?? sections.get('Summary') ?? (typeof reading.summary === 'string' ? reading.summary : undefined), ideaText = sections.get('Ideas') ?? sections.get('Debate'), quoteText = sections.get('Quotes');
+    if (form === 'digest')
+        parseJungleNote(markdown);
+    const gist = gistText?.replace(/\s+/g, ' ').trim(), ideas = ideaText === undefined ? undefined : loose(() => ideaList(ideaText)), quotes = quoteText === undefined ? undefined : loose(() => quoteList(quoteText));
+    return { schemaVersion: 1, source: 'jungle-reading', form, slug, status: typeof reading.status === 'string' ? reading.status : '', title: typeof meta.title === 'string' ? meta.title : '', sourceUrl: typeof reading.source_url === 'string' ? reading.source_url : '', capture, frontmatter: meta, ...(gist ? { gist } : {}), ...(ideas?.length ? { ideas } : {}), ...(quotes?.length && quotes.every(q => q.text && q.speaker) ? { quotes } : {}), markdownSha256: createHash('sha256').update(markdown).digest('hex'), markdown };
+}
+const stop = new Set('the a an and or but of to in on for with by from at as is are was were be been it its this that these those not no than then so such into about over under more most less can could would should will just also only very their there they them we our you your he she his her i my me what which who how when where why do does did has have had'.split(' '));
+/** Content-word stems: lowercase, stopwords out, a crude plural/tense strip. Shared by idea assignment and lexical coverage. */
+export function contentTerms(text) {
+    return new Set(foldText(text).toLowerCase().split(/[^a-z0-9]+/).filter(w => w.length > 2 && !stop.has(w)).map(w => w.replace(/(?:ing|ed|es|s)$/, '')).filter(w => w.length > 2));
+}
+export function termSimilarity(a, b) { if (!a.size || !b.size)
+    return 0; let n = 0; for (const t of a)
+    if (b.has(t))
+        n++; return n / Math.sqrt(a.size * b.size); }
+/** Best-first assignment: every idea gets at most one quote before any gets a second; zero-overlap pairs are never made. */
+export function assignQuotes(ideas, quotes) { return assign(ideas, quotes); }
+function assign(ideas, quotes) {
+    const ideaTerms = ideas.map(i => contentTerms(`${i.claim} ${i.detail}`)), pairs = [];
+    quotes.forEach((q, qi) => { const t = contentTerms(q.text); ideaTerms.forEach((it, ii) => { const s = termSimilarity(t, it); if (s > 0)
+        pairs.push({ q: qi, i: ii, s }); }); });
+    pairs.sort((a, b) => b.s - a.s || Math.abs(a.q - a.i) - Math.abs(b.q - b.i) || a.q - b.q);
+    const out = quotes.map(() => null), load = ideas.map(() => 0);
+    for (const cap of [1, L.maxQuotesPerIdea])
+        for (const p of pairs)
+            if (out[p.q] === null && load[p.i] < cap) {
+                out[p.q] = p.i;
+                load[p.i]++;
+            }
+    return out;
+}
+export const firstSentence = (text) => { const m = /^(.+?[.!?]["”’)]*)\s+(?=["“(]?[A-Z0-9])/.exec(text); return m ? [m[1], text.slice(m[0].length)] : [text, '']; };
+export function formOf(note) {
+    if (note.tags.includes('video'))
+        return 'video';
+    if (note.tags.includes('x'))
+        return 'thread';
+    return /\.pdf(?:$|[?#])|arxiv\.org\/pdf\//i.test(note.sourceUrl) ? 'pdf' : 'article';
+}
+/** Convert one Jungle digest note. With `capture` (the capture Markdown, frontmatter allowed), quotes are located and the gold is checked. */
+export function convertJungleDigest(noteMarkdown, capture) {
+    const note = parseJungleNote(noteMarkdown), lossy = new Set(), source = capture === undefined ? undefined : captureBody(capture);
+    const [lead, rest] = firstSentence(note.gist);
+    let context = '';
+    for (let r = rest; r;) {
+        const [s, next] = firstSentence(r);
+        if (words(context ? `${context} ${s}` : s) > L.contextWords) {
+            lossy.add('context-trimmed');
+            break;
+        }
+        context = context ? `${context} ${s}` : s;
+        r = next;
+    }
+    const assigned = assign(note.ideas, note.quotes), authors = new Set(note.authors.map(a => foldText(a).toLowerCase())), form = formOf(note);
+    const located = note.quotes.map(q => source === undefined ? undefined : snapQuote(source, q.text));
+    const ideas = note.ideas.map((idea, i) => {
+        const mine = note.quotes.map((q, k) => ({ q, k })).filter(x => assigned[x.k] === i);
+        const other = mine.find(x => !authors.has(foldText(x.q.speaker).toLowerCase()));
+        // A non-author in a talk is a speaker; in writing, the author reports them.
+        const voice = other ? { type: form === 'video' ? 'speaker' : 'reported', name: other.q.speaker } : { type: 'author' };
+        const quotes = mine.map(x => ({ text: located[x.k]?.text ?? x.q.text, speaker: x.q.speaker }));
+        return { claim: idea.claim, detail: idea.detail, kind: 'argument', voice, quotes };
+    });
+    const content = note.tags.filter(t => t !== 'reading' && !jungleFormatTags.includes(t)), tags = (content.length ? content : note.tags.filter(t => t !== 'reading')).slice(0, L.maxTags);
+    if (!content.length)
+        lossy.add('no-content-tag');
+    if (assigned.some(a => a === null))
+        lossy.add('quote-unassigned');
+    if (located.some(l => l?.match === 'missing'))
+        lossy.add('quote-missing');
+    if (ideas.some(i => !i.quotes.length))
+        lossy.add('idea-unquoted');
+    const gold = { schemaVersion: 1, form, gist: { lead, ...(context ? { context } : {}) }, ideas, tags };
+    const options = { tags: jungleTags };
+    const issues = source === undefined ? [] : checkDigest(gold, source, options).issues;
+    let importable;
+    if (source !== undefined) {
+        const kept = note.quotes.map((_, k) => assigned[k] !== null && located[k]?.match !== 'missing');
+        // Import keeps Sponge's ordering rule: ideas by their earliest quote in the capture, quotes of at least minQuoteWords.
+        const at = (t) => locateAll(source, t)[0]?.start ?? Number.MAX_SAFE_INTEGER;
+        const trimmed = { ...gold, ideas: note.ideas.map((_, i) => ({ ...ideas[i], quotes: note.quotes.flatMap((__, k) => assigned[k] === i && kept[k] && words(located[k].text) >= minQuoteWords ? [{ text: located[k].text, speaker: note.quotes[k].speaker }] : []).sort((a, b) => at(a.text) - at(b.text)) })).filter(i => i.quotes.length).sort((a, b) => at(a.quotes[0].text) - at(b.quotes[0].text)) };
+        if (checkDigest(trimmed, source, options).ok)
+            importable = trimmed;
+    }
+    return { note, gold, quotes: note.quotes.map((_, k) => ({ idea: assigned[k] ?? null, ...(located[k] ? { match: located[k].match } : {}) })), lossy: [...lossy], issues, ...(importable ? { importable } : {}), clean: source !== undefined && issues.length === 0 && lossy.size === 0 };
+}

package/dist/src/importers/jungle-store.d.ts

@@ -0,0 +1,45 @@
+import type { DatabaseSync } from 'node:sqlite';
+import { type DigestV1 } from '../digest/schema.js';
+import { type VisibilityState } from '../identity/visibility.js';
+import { type JungleEditorial } from './jungle-digest.js';
+/**
+ * The library side of `sponge import jungle`: one Jungle Reading note onto the item holding its capture.
+ * `planJungleNote` only reads; `applyJungleNote` runs the same plan inside the caller's transaction, through the logged paths.
+ */
+export declare const jungleNamespace = "jungle", jungleRealm = "import:jungle", jungleProducer: {
+    readonly importer: "sponge-jungle-reading";
+    readonly version: "1";
+};
+export type EditorialAction = 'new' | 'unchanged' | 'user-review-wins';
+export type DigestAction = 'new' | 'unchanged' | 'not-convertible' | 'not-digest' | 'user-review-wins' | 'user-digest-wins';
+export type VisibilityAction = 'set-public' | 'set-private' | 'restore-default' | 'unchanged' | 'kept-user-edit' | 'guarded' | 'default-policy';
+export interface JungleNoteInput {
+    itemId: string;
+    revisionId: string;
+    editorial: JungleEditorial;
+    v1?: DigestV1;
+}
+export interface JunglePlan {
+    editorialId: string;
+    editorial: EditorialAction;
+    v1Id?: string;
+    v1: DigestAction;
+    visibility: VisibilityAction;
+}
+/** What Jungle's `reading.status` asks for: `published` is Jungle's explicit public approval, `draft` stays private. */
+export declare function desiredVisibility(status: string): 'public' | 'private' | undefined;
+/**
+ * Jungle's decision becomes an explicit override only while nobody else has set one: once the latest override was not
+ * written by this importer, the user's choice wins. A public decision never lifts the capability-URL or local-file guards.
+ */
+export declare function visibilityAction(state: Pick<VisibilityState, 'access' | 'default' | 'override'>, status: string, userOverride: boolean): {
+    action: VisibilityAction;
+    target?: 'public' | 'private' | null;
+};
+export declare function planJungleNote(db: DatabaseSync, input: JungleNoteInput): JunglePlan;
+/**
+ * Apply one note; the caller holds the write transaction. A new editorial object is accepted through `review.note` unless
+ * the user already rejected or cleared an earlier one; a clean Digest v1 is stored and accepted per field, linked to the
+ * editorial object by its input hash, unless the user owns a digest of this item or changed an imported digest's review.
+ */
+export declare function applyJungleNote(db: DatabaseSync, input: JungleNoteInput): JunglePlan;

package/dist/src/importers/jungle-store.js

@@ -0,0 +1,95 @@
+import { canonicalDigest, derivationId, putDigest, reviewDigest } from '../digest/store.js';
+import { canonicalEditorial, putEditorial, reviewEditorial } from '../digest/editorial.js';
+import { parseDigest } from '../digest/schema.js';
+import { visibility as visibilityState, setVisibility } from '../identity/visibility.js';
+import { digest } from '../store/files.js';
+import { jungleTags } from './jungle-digest.js';
+/**
+ * The library side of `sponge import jungle`: one Jungle Reading note onto the item holding its capture.
+ * `planJungleNote` only reads; `applyJungleNote` runs the same plan inside the caller's transaction, through the logged paths.
+ */
+export const jungleNamespace = 'jungle', jungleRealm = 'import:jungle', jungleProducer = { importer: 'sponge-jungle-reading', version: '1' };
+/** Receipts of the importer's own visibility writes (local, like every import receipt), so a later user override is recognisable. */
+const visibilityReceipts = 'jungle-visibility';
+/** What Jungle's `reading.status` asks for: `published` is Jungle's explicit public approval, `draft` stays private. */
+export function desiredVisibility(status) { return status === 'published' ? 'public' : status === 'draft' ? 'private' : undefined; }
+/**
+ * Jungle's decision becomes an explicit override only while nobody else has set one: once the latest override was not
+ * written by this importer, the user's choice wins. A public decision never lifts the capability-URL or local-file guards.
+ */
+export function visibilityAction(state, status, userOverride) {
+    const desired = desiredVisibility(status);
+    if (!desired)
+        return { action: 'default-policy' };
+    if (userOverride)
+        return { action: 'kept-user-edit' };
+    if (desired === 'public' && state.access !== 'imported' && state.access !== 'public')
+        return { action: 'guarded' };
+    const target = desired === state.default ? null : desired;
+    if ((state.override ?? null) === target)
+        return { action: 'unchanged' };
+    return { action: target === null ? 'restore-default' : target === 'public' ? 'set-public' : 'set-private', target };
+}
+const allAccepted = (o) => ({ gist: 'accepted', ideas: o.ideas.map(() => 'accepted'), quotes: o.ideas.map(i => i.quotes.map(() => 'accepted')) });
+const imported = (r) => { try {
+    return JSON.parse(String(r.producer_json))?.importer === jungleProducer.importer;
+}
+catch {
+    return false;
+} };
+const latestVisibility = (db, itemId) => db.prepare("SELECT id FROM changes WHERE entity='item' AND entity_id=? AND field='visibility' ORDER BY seq DESC LIMIT 1").get(itemId);
+/** The latest logged visibility write was not this importer's: the user chose, even when the choice was `default`. */
+function userOverride(db, itemId) {
+    const latest = latestVisibility(db, itemId);
+    if (!latest)
+        return (db.prepare('SELECT visibility FROM items WHERE id=?').get(itemId)?.visibility ?? null) !== null;
+    return !db.prepare('SELECT 1 FROM import_receipts WHERE namespace=? AND source_key=? AND source_hash=?').get(visibilityReceipts, itemId, digest(String(latest.id)));
+}
+export function planJungleNote(db, input) {
+    const { itemId, revisionId, editorial } = input, editorialId = derivationId(revisionId, 'editorial', editorial.markdownSha256, canonicalEditorial(editorial));
+    const editorials = db.prepare("SELECT id,review_json FROM derivations WHERE item_id=? AND kind='editorial'").all(itemId);
+    const editorialAction = editorials.some(r => r.id === editorialId) ? 'unchanged' : editorials.some(r => JSON.parse(String(r.review_json)).note !== 'accepted') ? 'user-review-wins' : 'new';
+    let v1 = 'not-digest', v1Id;
+    if (editorial.form === 'digest' && !input.v1)
+        v1 = 'not-convertible';
+    else if (input.v1) {
+        const digests = db.prepare("SELECT id,producer_json,output_json,review_json FROM derivations WHERE item_id=? AND kind='digest'").all(itemId), parsed = parseDigest(input.v1).digest;
+        v1Id = parsed && derivationId(revisionId, 'digest', editorialId, canonicalDigest(parsed));
+        if (!parsed)
+            v1 = 'not-convertible';
+        else if (digests.some(r => r.id === v1Id))
+            v1 = 'unchanged';
+        else if (digests.some(r => !imported(r)))
+            v1 = 'user-digest-wins';
+        else if (digests.some(r => JSON.stringify(JSON.parse(String(r.review_json))) !== JSON.stringify(allAccepted(JSON.parse(String(r.output_json))))))
+            v1 = 'user-review-wins';
+        else
+            v1 = editorialAction === 'user-review-wins' ? 'user-review-wins' : 'new';
+    }
+    const visibility = visibilityAction(visibilityState(db, itemId), editorial.status, userOverride(db, itemId)).action;
+    return { editorialId, editorial: editorialAction, ...(v1Id ? { v1Id } : {}), v1, visibility };
+}
+/**
+ * Apply one note; the caller holds the write transaction. A new editorial object is accepted through `review.note` unless
+ * the user already rejected or cleared an earlier one; a clean Digest v1 is stored and accepted per field, linked to the
+ * editorial object by its input hash, unless the user owns a digest of this item or changed an imported digest's review.
+ */
+export function applyJungleNote(db, input) {
+    const plan = planJungleNote(db, input), producer = { ...jungleProducer, note: input.editorial.slug };
+    if (plan.editorial !== 'unchanged') {
+        putEditorial(db, { itemId: input.itemId, revisionId: input.revisionId, producer, output: input.editorial });
+        if (plan.editorial === 'new')
+            reviewEditorial(db, plan.editorialId, 'accepted');
+    }
+    if (plan.v1 === 'new') {
+        const stored = putDigest(db, { itemId: input.itemId, revisionId: input.revisionId, producer: { ...producer, editorial: plan.editorialId }, inputHash: plan.editorialId, output: input.v1, options: { tags: jungleTags } });
+        reviewDigest(db, stored.id, allAccepted(input.v1));
+    }
+    const action = visibilityAction(visibilityState(db, input.itemId), input.editorial.status, userOverride(db, input.itemId));
+    if (action.target !== undefined) {
+        setVisibility(db, input.itemId, action.target);
+        const latest = latestVisibility(db, input.itemId);
+        db.prepare('INSERT OR IGNORE INTO import_receipts VALUES(?,?,?,?,?,?)').run(visibilityReceipts, input.itemId, digest(String(latest.id)), jungleProducer.version, input.itemId, 'visibility');
+    }
+    return plan;
+}

package/dist/src/importers/jungle.d.ts

@@ -0,0 +1,54 @@
+import type { LibraryStore } from '../store/index.js';
+import { type DigestV1 } from '../digest/schema.js';
+import { type JungleEditorial } from './jungle-digest.js';
+import { type DigestAction, type EditorialAction, type VisibilityAction } from './jungle-store.js';
+import type { BundleVerifier } from './inspect.js';
+/**
+ * `sponge import jungle DIR`: every Jungle Reading note (`kb/notes/reading/<slug>.md`) onto the item holding its capture.
+ * The capture goes through the verified bundle import (always as a private association); the note becomes an accepted
+ * editorial derivation, plus an accepted Digest v1 when it converts with nothing dropped; `reading.status` becomes the
+ * item's visibility. Jungle is only read. A dry run reads the notes, captures and library and writes nothing.
+ */
+export interface JungleImportOptions {
+    verify?: BundleVerifier;
+    dryRun?: boolean;
+    limit?: number;
+    offset?: number;
+    signal?: AbortSignal;
+}
+export type JungleSkip = 'not-reading-note' | 'malformed-note' | 'no-capture' | 'capture-missing' | 'item-deleted' | 'capture-unsupported' | 'capture-failed';
+export interface JungleEntry {
+    slug: string;
+    form?: string;
+    status?: string;
+    item?: 'new' | 'existing';
+    itemId?: string;
+    editorial?: EditorialAction;
+    v1?: DigestAction;
+    v1Issues?: string[];
+    visibility?: VisibilityAction;
+    skipped?: JungleSkip;
+    code?: string;
+}
+export interface JungleImportReport {
+    schemaVersion: 1;
+    dryRun: boolean;
+    captureVerification: 'deferred' | 'verified';
+    notes: number;
+    counts: {
+        newItems: number;
+        existingItems: number;
+        editorial: Record<string, number>;
+        v1: Record<string, number>;
+        visibility: Record<string, number>;
+        skipped: Record<string, number>;
+    };
+    entries: JungleEntry[];
+    nextOffset: number | null;
+}
+/** Digest v1 only when the note converts with nothing dropped or trimmed and passes every rule against this revision's Markdown. */
+export declare function jungleV1(editorial: JungleEditorial, markdown: string): {
+    v1?: DigestV1;
+    issues: string[];
+};
+export declare function importJungle(store: LibraryStore, jungle: string, options?: JungleImportOptions): Promise<JungleImportReport>;

package/dist/src/importers/jungle.js

@@ -0,0 +1,134 @@
+import { readdir, realpath } from 'node:fs/promises';
+import { dirname, isAbsolute, join } from 'node:path';
+import { readConfined, hash } from '../acquisition/files.js';
+import { object } from '../acquisition/normalize.js';
+import { localizeMarkdown } from '../app/markdown.js';
+import { checkDigest } from '../digest/schema.js';
+import { accessClass, defaultVisibility } from '../identity/uri.js';
+import { convertJungleDigest, parseJungleEditorial, jungleTags, JungleDigestError } from './jungle-digest.js';
+import { jungleNamespace, jungleRealm, visibilityAction } from './jungle-store.js';
+import { importBundleEntry, importBundleHash } from './apply.js';
+const noteBytes = 4 * 1024 ** 2, captureBytes = 16 * 1024 ** 2;
+const errorCode = (error) => String(error?.code ?? (error instanceof Error ? error.message : 'unclassified'));
+/** A capture link is `<dir>/<name>` under `kb/`, never escaping it; the bundle is its directory. */
+function captureLink(link) { if (isAbsolute(link) || link.includes('\\') || link.split('/').length < 2 || link.split('/').some(p => !p || p === '.' || p === '..'))
+    throw new JungleDigestError('no-capture'); return link; }
+/** Digest v1 only when the note converts with nothing dropped or trimmed and passes every rule against this revision's Markdown. */
+export function jungleV1(editorial, markdown) {
+    if (editorial.form !== 'digest')
+        return { issues: [] };
+    const c = convertJungleDigest(editorial.markdown, markdown), check = checkDigest(c.gold, markdown, { tags: jungleTags }), issues = [...c.lossy, ...c.issues.map(i => `${i.path}:${i.code}`), ...check.issues.map(i => `${i.path}:${i.code}`)];
+    return c.clean && check.ok ? { v1: c.gold, issues: [] } : { issues: [...new Set(issues)] };
+}
+/** What a verified import would store, read from the bundle without the verifier: the same localization, from the manifest's asset list. */
+async function previewCapture(kb, bundle, link) {
+    const manifest = object(JSON.parse((await readConfined(kb, join(bundle, 'capture.json'), captureBytes)).toString('utf8'))), pdf = manifest.kind === 'pdf';
+    const listed = (pdf ? (Array.isArray(manifest.images) ? manifest.images.map(i => object(i).asset) : []) : Array.isArray(manifest.assets) ? manifest.assets : []).flatMap(raw => { const a = raw && typeof raw === 'object' ? raw : {}; return typeof a.path === 'string' && typeof a.sha256 === 'string' ? [{ relativePath: a.path, sha256: a.sha256, bytes: Number(a.bytes), mediaType: String(a.mimeType) }] : []; });
+    const text = (await readConfined(kb, link + '.md', captureBytes)).toString('utf8'), markdown = localizeMarkdown(pdf ? text.replace(/^<!-- pdf-page: ([1-9][0-9]*) -->$/gm, '## Page $1') : text, listed).markdown;
+    const locator = pdf ? String(object(manifest.source ?? {}).requestedUrl ?? '') : String(manifest.sourceUrl ?? '');
+    return { markdown, locator };
+}
+export async function importJungle(store, jungle, options = {}) {
+    if (!isAbsolute(jungle))
+        throw Object.assign(Error('invalid-input'), { code: 'invalid-input' });
+    const kb = await realpath(join(jungle, 'kb')), notesDir = join(kb, 'notes', 'reading'), files = (await readdir(notesDir)).filter(f => /^[a-z0-9][a-z0-9-]*\.md$/.test(f)).sort();
+    const limit = options.limit ?? 1000, offset = options.offset ?? 0, dryRun = options.dryRun !== false;
+    if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1000 || !Number.isSafeInteger(offset) || offset < 0 || offset > files.length)
+        throw Object.assign(Error('invalid-input'), { code: 'invalid-input' });
+    if (!dryRun && !options.verify)
+        throw Object.assign(Error('runtime-missing'), { code: 'runtime-missing' });
+    const report = { schemaVersion: 1, dryRun, captureVerification: dryRun ? 'deferred' : 'verified', notes: files.length, counts: { newItems: 0, existingItems: 0, editorial: {}, v1: {}, visibility: {}, skipped: {} }, entries: [], nextOffset: offset + limit < files.length ? offset + limit : null };
+    const bump = (m, k) => { m[k] = (m[k] ?? 0) + 1; }, seen = new Set();
+    for (const file of files.slice(offset, offset + limit)) {
+        if (options.signal?.aborted)
+            throw Object.assign(Error('cancelled'), { code: 'cancelled' });
+        const slug = file.slice(0, -3);
+        let entry = { slug };
+        try {
+            let editorial;
+            try {
+                editorial = parseJungleEditorial((await readConfined(notesDir, file, noteBytes)).toString('utf8'));
+            }
+            catch (error) {
+                const code = error instanceof JungleDigestError ? error.code : 'malformed';
+                entry.skipped = code === 'not-digest' ? 'not-reading-note' : code === 'no-capture' ? 'no-capture' : 'malformed-note';
+                throw entry;
+            }
+            entry = { slug, form: editorial.form, status: editorial.status };
+            let link;
+            try {
+                link = captureLink(editorial.capture);
+            }
+            catch {
+                entry.skipped = 'no-capture';
+                throw entry;
+            }
+            const bundle = dirname(link);
+            try {
+                await readConfined(kb, join(bundle, 'capture.json'), captureBytes);
+            }
+            catch {
+                entry.skipped = 'capture-missing';
+                throw entry;
+            }
+            const prior = store.importedItem(jungleRealm, hash(JSON.stringify([jungleNamespace, link])));
+            if (prior?.deleted) {
+                entry.skipped = 'item-deleted';
+                entry.itemId = prior.itemId;
+                throw entry;
+            }
+            // Two notes on one capture share its item, in a dry run too.
+            entry.item = prior || seen.has(link) ? 'existing' : 'new';
+            seen.add(link);
+            if (dryRun) {
+                if (prior?.revisionId) {
+                    const markdown = store.revisionMarkdown(prior.revisionId), { v1, issues } = jungleV1(editorial, markdown), plan = store.planJungleNote({ itemId: prior.itemId, revisionId: prior.revisionId, editorial, ...(v1 ? { v1 } : {}) });
+                    Object.assign(entry, { itemId: prior.itemId, editorial: plan.editorial, v1: plan.v1, ...(issues.length ? { v1Issues: issues } : {}), visibility: plan.visibility });
+                }
+                else {
+                    const preview = await previewCapture(kb, bundle, link), { v1, issues } = jungleV1(editorial, preview.markdown), access = accessClass(jungleRealm, preview.locator);
+                    Object.assign(entry, { editorial: 'new', v1: editorial.form !== 'digest' ? 'not-digest' : v1 ? 'new' : 'not-convertible', ...(issues.length ? { v1Issues: issues } : {}), visibility: visibilityAction({ access, default: defaultVisibility(access, 'private'), override: null }, editorial.status, false).action });
+                }
+            }
+            else {
+                let imported;
+                try {
+                    imported = await importBundleEntry(store, kb, jungleNamespace, { id: link, bundle, privacy: 'private', notes: [] }, options.verify, false, options.signal);
+                }
+                catch (error) {
+                    const code = errorCode(error);
+                    if (['cleanup-required', 'owner-busy', 'cancelled'].includes(code))
+                        throw error;
+                    entry.skipped = code === 'unsupported' ? 'capture-unsupported' : 'capture-failed';
+                    entry.code = code;
+                    throw entry;
+                }
+                const itemId = imported.itemId, revisionId = store.importedRevision(itemId, importBundleHash(imported.sourceHash)), markdown = store.revisionMarkdown(revisionId), { v1, issues } = jungleV1(editorial, markdown);
+                const plan = store.applyJungleNote({ itemId, revisionId, editorial, ...(v1 ? { v1 } : {}) });
+                Object.assign(entry, { itemId, editorial: plan.editorial, v1: plan.v1, ...(issues.length ? { v1Issues: issues } : {}), visibility: plan.visibility });
+            }
+            bump(report.counts.editorial, entry.editorial);
+            bump(report.counts.v1, entry.v1);
+            bump(report.counts.visibility, entry.visibility);
+            if (entry.item === 'new')
+                report.counts.newItems++;
+            else
+                report.counts.existingItems++;
+            report.entries.push(entry);
+        }
+        catch (error) {
+            if (error !== entry) {
+                const code = errorCode(error);
+                if (['cleanup-required', 'owner-busy', 'cancelled'].includes(code)) {
+                    report.nextOffset = offset + report.entries.length;
+                    break;
+                }
+                entry.skipped = 'capture-failed';
+                entry.code = code;
+            }
+            bump(report.counts.skipped, entry.skipped);
+            report.entries.push(entry);
+        }
+    }
+    return report;
+}

package/dist/src/indexing/index.d.ts

@@ -23,6 +23,13 @@ export declare class IndexError extends Error {
     code: 'invalid-index' | 'stale-index' | 'incomplete-index' | 'cancelled-index';
     constructor(code: 'invalid-index' | 'stale-index' | 'incomplete-index' | 'cancelled-index');
 }
+export type VectorEncoding = 'f32' | 'int8';
+/** Storage format follows the space identity: no `quantization` (legacy) or `f32` stores float32 LE; `int8-symmetric-maxabs-v1` stores one int8 code per dimension. */
+export declare function spaceEncoding(identity: Record<string, unknown>): VectorEncoding;
+/** int8 codes are round(x*127/max|x|) of a unit vector. The per-vector scale is not stored: cosine is scale-invariant, so codes rank directly and dequantize by renormalizing. */
+export declare function encodeVector(vector: readonly number[], encoding: VectorEncoding): Buffer;
+/** Exact byte length, finite floats, int8 codes within [-127,127] and nonzero; int8 returns the unit dequantized vector. */
+export declare function decodeVector(bytes: Uint8Array, dimensions: number, encoding: VectorEncoding): number[];
 /** Unicode-safe chunks fit the fixed document prompt within the embedding text budget, preferring whitespace breaks. */
 export declare function chunkDocument(text: string): string[];
 /** Identity must already be the verified embedder fingerprint. Existing identities are immutable. */