@doxbrix/doxloop 0.1.4 → 0.2.0

package/dist/sync.d.ts

@@ -1,7 +1,42 @@
 import type { SourceBinding, SourceChange, SyncState } from './types.js';
+/**
+ * Baseline marker for a local folder that is not a Git repository. Such a
+ * source is compared by content: the recorded per-file digests name exactly
+ * which files changed, so drift can still be traced to individual pages.
+ */
+export declare const LOCAL_CONTENT_BASELINE = "local-content";
+/**
+ * Baseline marker for a crawled documentation-site snapshot. The snapshot is
+ * frozen, so a change means the site was re-crawled and the recorded per-page
+ * digests name exactly which existing pages changed.
+ */
+export declare const DOCS_SITE_BASELINE = "docs-site";
 export declare const SYNC_STATE_FILE: string;
 export declare function readSyncState(root: string): Promise<SyncState>;
 export declare function recordSyncState(root: string, sources: SourceBinding[]): Promise<SyncState>;
 export declare function collectSourceChanges(root: string, sources: SourceBinding[]): Promise<SourceChange[]>;
+/** Name-status lines, in Git's format, describing how one manifest became another. */
+export declare function manifestChanges(before: Record<string, string>, after: Record<string, string>): string[];
+/** Stable content fingerprints used to invalidate an approved planning checkpoint. */
+export declare function sourceSnapshotFingerprints(root: string, sources: SourceBinding[]): Promise<Record<string, string | null>>;
+/**
+ * Source-relative paths touched by a change, parsed from the same Git output
+ * that produced the human-readable lists. Rename and copy entries contribute
+ * both the old and the new path so either one can match documented evidence.
+ */
+export declare function changedSourcePaths(change: SourceChange): string[];
 export declare function formatSourceChanges(changes: SourceChange[]): string;
+export interface LocalSourceManifest {
+    /** Source-relative portable paths to their content digests. */
+    files: Record<string, string>;
+    /** One digest over every path and content, in the historical format. */
+    fingerprint: string;
+}
+/**
+ * Every file that counts as source content, with its digest. Git repositories
+ * contribute tracked and untracked-but-not-ignored files; a plain folder is
+ * walked directly with the same exclusions a repository would apply by
+ * convention. Secrets are never read.
+ */
+export declare function localSourceManifest(path: string): Promise<LocalSourceManifest>;
 //# sourceMappingURL=sync.d.ts.map

package/dist/sync.js

@@ -1,11 +1,25 @@
 import { execFile } from 'node:child_process';
 import { createHash } from 'node:crypto';
-import { lstat, readFile, readlink, writeFile } from 'node:fs/promises';
-import { join, resolve } from 'node:path';
+import { lstat, readFile, readdir, readlink, writeFile } from 'node:fs/promises';
+import { basename, join, relative, resolve } from 'node:path';
 import { promisify } from 'node:util';
 import { pathExists, readJson } from './fs.js';
+import { diffOpenApi, loadOpenApiSource, openApiChangedIdentifiers } from './openapi.js';
 import { isSpecUrl, sourceKind } from './project.js';
+import { remoteHead } from './remote-source.js';
 const SPEC_BASELINE = 'openapi-spec';
+/**
+ * Baseline marker for a local folder that is not a Git repository. Such a
+ * source is compared by content: the recorded per-file digests name exactly
+ * which files changed, so drift can still be traced to individual pages.
+ */
+export const LOCAL_CONTENT_BASELINE = 'local-content';
+/**
+ * Baseline marker for a crawled documentation-site snapshot. The snapshot is
+ * frozen, so a change means the site was re-crawled and the recorded per-page
+ * digests name exactly which existing pages changed.
+ */
+export const DOCS_SITE_BASELINE = 'docs-site';
 export const SYNC_STATE_FILE = join('.doxloop', 'sync-state.json');
 const MAX_LISTED_FILES = 40;
 const run = promisify(execFile);
@@ -23,26 +37,58 @@ export async function recordSyncState(root, sources) {
     const state = { schemaVersion: 1, sources: {} };
     for (const source of sources) {
         if (sourceKind(source) === 'openapi') {
-            if (isSpecUrl(source.path))
+            const loaded = await loadOpenApiSource(root, source);
+            state.sources[source.name] = {
+                commit: SPEC_BASELINE,
+                recordedAt: new Date().toISOString(),
+                contentFingerprint: loaded.hash,
+                connector: {
+                    id: 'openapi',
+                    version: 1,
+                    ...(loaded.etag ? { etag: loaded.etag } : {}),
+                    ...(loaded.lastModified ? { lastModified: loaded.lastModified } : {}),
+                    openapi: loaded.snapshot,
+                },
+            };
+            continue;
+        }
+        if (source.remote) {
+            state.sources[source.name] = {
+                commit: await remoteHead(source.remote),
+                recordedAt: new Date().toISOString(),
+            };
+            continue;
+        }
+        const sourcePath = resolve(root, source.path);
+        if (sourceKind(source) === 'docs-site') {
+            if (!(await pathExists(sourcePath)))
                 continue;
-            const fingerprint = await specFileFingerprint(resolve(root, source.path));
-            if (fingerprint) {
-                state.sources[source.name] = {
-                    commit: SPEC_BASELINE,
-                    recordedAt: new Date().toISOString(),
-                    contentFingerprint: fingerprint,
-                };
-            }
+            const manifest = await localSourceManifest(sourcePath);
+            state.sources[source.name] = {
+                commit: DOCS_SITE_BASELINE,
+                recordedAt: new Date().toISOString(),
+                contentFingerprint: manifest.fingerprint,
+                files: manifest.files,
+            };
             continue;
         }
-        const commit = await headCommit(resolve(root, source.path));
+        const commit = await headCommit(sourcePath);
         if (commit) {
-            const sourcePath = resolve(root, source.path);
             state.sources[source.name] = {
                 commit,
                 recordedAt: new Date().toISOString(),
                 contentFingerprint: await sourceContentFingerprint(sourcePath),
             };
+            continue;
+        }
+        if (await pathExists(sourcePath)) {
+            const manifest = await localSourceManifest(sourcePath);
+            state.sources[source.name] = {
+                commit: LOCAL_CONTENT_BASELINE,
+                recordedAt: new Date().toISOString(),
+                contentFingerprint: manifest.fingerprint,
+                files: manifest.files,
+            };
         }
     }
     await writeFile(join(root, SYNC_STATE_FILE), `${JSON.stringify(state, null, 2)}\n`, 'utf8');
@@ -61,9 +107,23 @@ export async function collectSourceChanges(root, sources) {
             changes.push({ ...source, kind: 'missing-path' });
             continue;
         }
+        if (sourceKind(source) === 'docs-site') {
+            changes.push(await collectLocalContentChange(source, path, state, DOCS_SITE_BASELINE));
+            continue;
+        }
         const head = await headCommit(path);
         if (!head) {
-            changes.push({ ...source, kind: 'not-git' });
+            // A remote source materialized from a provider is an immutable snapshot
+            // pinned at the recorded commit: the folder is named after that commit
+            // and nothing inside it can drift, so it is current by construction.
+            // Treating it as a baseline-free plain folder made every page of a
+            // GitHub-sourced project count as unverified.
+            const record = state.sources[source.name];
+            if (source.remote && record && record.commit !== LOCAL_CONTENT_BASELINE && basename(path) === record.commit) {
+                changes.push({ ...source, kind: 'unchanged', baseline: record.commit, head: record.commit, changedFiles: [], uncommittedFiles: [] });
+                continue;
+            }
+            changes.push(await collectLocalContentChange(source, path, state));
             continue;
         }
         const uncommittedFiles = await uncommittedChanges(path);
@@ -95,32 +155,150 @@ export async function collectSourceChanges(root, sources) {
     }
     return changes;
 }
+/**
+ * A folder without Git history is compared against the file digests recorded
+ * at the last sync, so a scheduled check on a plain folder still names the
+ * files that changed.
+ */
+async function collectLocalContentChange(source, path, state, baseline = LOCAL_CONTENT_BASELINE) {
+    const record = state.sources[source.name];
+    if (!record || record.commit !== baseline || !record.files) {
+        return { ...source, kind: 'no-baseline', head: baseline, uncommittedFiles: [] };
+    }
+    const manifest = await localSourceManifest(path);
+    const changedFiles = record.contentFingerprint === manifest.fingerprint
+        ? []
+        : manifestChanges(record.files, manifest.files);
+    return {
+        ...source,
+        kind: changedFiles.length === 0 ? 'unchanged' : 'changed',
+        baseline,
+        head: baseline,
+        changedFiles,
+        uncommittedFiles: [],
+    };
+}
+/** Name-status lines, in Git's format, describing how one manifest became another. */
+export function manifestChanges(before, after) {
+    const lines = [];
+    for (const file of Object.keys(after).sort()) {
+        if (!(file in before))
+            lines.push(`A\t${file}`);
+        else if (before[file] !== after[file])
+            lines.push(`M\t${file}`);
+    }
+    for (const file of Object.keys(before).sort()) {
+        if (!(file in after))
+            lines.push(`D\t${file}`);
+    }
+    return lines;
+}
+/** Stable content fingerprints used to invalidate an approved planning checkpoint. */
+export async function sourceSnapshotFingerprints(root, sources) {
+    const fingerprints = {};
+    for (const source of sources) {
+        if (sourceKind(source) === 'openapi') {
+            try {
+                fingerprints[source.name] = (await loadOpenApiSource(root, source)).hash;
+            }
+            catch {
+                fingerprints[source.name] = null;
+            }
+            continue;
+        }
+        try {
+            fingerprints[source.name] = await sourceContentFingerprint(resolve(root, source.path));
+        }
+        catch {
+            fingerprints[source.name] = null;
+        }
+    }
+    return fingerprints;
+}
 async function collectSpecChange(root, source, state) {
-    if (isSpecUrl(source.path))
-        return { ...source, kind: 'spec-remote' };
-    const fingerprint = await specFileFingerprint(resolve(root, source.path));
-    if (!fingerprint)
-        return { ...source, kind: 'missing-path' };
     const record = state.sources[source.name];
-    if (record?.contentFingerprint === fingerprint) {
-        return { ...source, kind: 'spec-unchanged' };
+    let loaded;
+    try {
+        loaded = await loadOpenApiSource(root, source, record?.connector ? {
+            ...(record.connector.etag ? { etag: record.connector.etag } : {}),
+            ...(record.connector.lastModified ? { lastModified: record.connector.lastModified } : {}),
+        } : undefined);
     }
-    return { ...source, kind: 'spec-changed' };
+    catch (error) {
+        if (!isSpecUrl(source.path) && !(await pathExists(resolve(root, source.path)))) {
+            return { ...source, kind: 'missing-path' };
+        }
+        throw error;
+    }
+    if (record?.contentFingerprint === loaded.hash || loaded.notModified) {
+        return { ...source, kind: 'spec-unchanged', head: loaded.hash, summary: loaded.summary };
+    }
+    const apiDiff = diffOpenApi(record?.connector?.openapi, loaded.snapshot);
+    return {
+        ...source,
+        kind: 'spec-changed',
+        ...(record?.contentFingerprint ? { baseline: record.contentFingerprint } : {}),
+        head: loaded.hash,
+        summary: loaded.summary,
+        apiDiff,
+        changedIdentifiers: openApiChangedIdentifiers(apiDiff),
+    };
+}
+/**
+ * Source-relative paths touched by a change, parsed from the same Git output
+ * that produced the human-readable lists. Rename and copy entries contribute
+ * both the old and the new path so either one can match documented evidence.
+ */
+export function changedSourcePaths(change) {
+    if (change.kind === 'spec-changed')
+        return change.changedIdentifiers;
+    if (change.kind !== 'changed' && change.kind !== 'no-baseline' && change.kind !== 'baseline-lost') {
+        return [];
+    }
+    const paths = new Set();
+    if (change.kind === 'changed') {
+        for (const line of change.changedFiles) {
+            for (const path of parseNameStatus(line))
+                paths.add(path);
+        }
+    }
+    for (const line of change.uncommittedFiles) {
+        for (const path of parsePorcelain(line))
+            paths.add(path);
+    }
+    return [...paths];
+}
+/** `M<tab>src/auth.ts` or `R100<tab>old.ts<tab>new.ts` */
+function parseNameStatus(line) {
+    const fields = line.split('\t');
+    return fields.slice(1).map(unquotePath).filter(Boolean);
 }
-async function specFileFingerprint(path) {
+/** `M src/auth.ts`, `?? new.ts`, or `R  old.ts -> new.ts`, already trimmed. */
+function parsePorcelain(line) {
+    const match = /^\S+\s+(.*)$/.exec(line);
+    if (!match?.[1])
+        return [];
+    return match[1]
+        .split(' -> ')
+        .map((part) => unquotePath(part))
+        .filter(Boolean);
+}
+function unquotePath(value) {
+    const path = value.trim();
+    if (!path.startsWith('"') || !path.endsWith('"'))
+        return path;
     try {
-        const content = await readFile(path);
-        return createHash('sha256').update(content).digest('hex');
+        return String(JSON.parse(path));
     }
     catch {
-        return undefined;
+        return path.slice(1, -1);
     }
 }
 export function formatSourceChanges(changes) {
     if (changes.length === 0)
         return '';
     const sections = changes.map((change) => {
-        const heading = `Source "${change.name}" (${change.path})`;
+        const heading = `Source "${change.name}" (${change.site?.url ?? change.path})`;
         switch (change.kind) {
             case 'missing-path':
                 return `${heading}: the configured path does not exist. Report this instead of guessing.`;
@@ -129,16 +307,35 @@ export function formatSourceChanges(changes) {
             case 'spec-unchanged':
                 return `${heading}: the API specification is unchanged since the last documentation sync.`;
             case 'spec-changed':
-                return `${heading}: the API specification changed since the last documentation sync, or no baseline is recorded yet. Compare it with the documented endpoints and update every affected page; the baseline is recorded when this task completes.`;
+                return `${heading}: the API specification changed${change.baseline ? ` (${short(change.baseline)} -> ${short(change.head)})` : ' and has no recorded baseline'}.
+${formatApiDelta(change.apiDiff)}
+Update only documentation affected by this structural delta; the new baseline is recorded after acceptance.`;
             case 'not-git':
                 return `${heading}: not a Git repository, so no change baseline is available. Inspect the source directly.`;
             case 'no-baseline':
+                if (change.head === DOCS_SITE_BASELINE) {
+                    return `${heading}: existing documentation site, crawled into the read-only snapshot at ${change.path}. No documentation sync baseline is recorded yet; read the snapshot's index.md and page files directly. A baseline is recorded when this task completes.`;
+                }
                 return withUncommitted(`${heading}: no documentation sync baseline is recorded yet. Inspect the source directly; a baseline is recorded when this task completes.`, change.uncommittedFiles);
             case 'baseline-lost':
                 return withUncommitted(`${heading}: the recorded baseline commit no longer exists (history rewritten?). Inspect the source directly; the baseline is re-recorded when this task completes.`, change.uncommittedFiles);
             case 'unchanged':
-                return withUncommitted(`${heading}: no source-content changes since the last documentation sync (${short(change.baseline)}).`, change.uncommittedFiles);
+                if (change.baseline === DOCS_SITE_BASELINE) {
+                    return `${heading}: the existing documentation snapshot at ${change.path} is unchanged since the last documentation sync.`;
+                }
+                return withUncommitted(`${heading}: no source-content changes since the last documentation sync (${change.baseline === LOCAL_CONTENT_BASELINE ? 'compared by content' : short(change.baseline)}).`, change.uncommittedFiles);
             case 'changed':
+                if (change.baseline === DOCS_SITE_BASELINE) {
+                    return `${heading}: the existing documentation site was re-crawled and these snapshot pages changed:\n${fileList(change.changedFiles)}\nRead the changed pages in the snapshot at ${change.path} and decide whether the rewritten documentation must follow.`;
+                }
+                if (change.remote) {
+                    return change.changedFiles.length > 0
+                        ? `${heading}: the remote repository changed (${short(change.baseline)} -> ${short(change.head)}).\nChanged files reported by ${change.remote.provider}:\n${fileList(change.changedFiles)}\nThe current commit was downloaded into this isolated read-only evidence snapshot; inspect files there directly.`
+                        : `${heading}: the remote repository changed (${short(change.baseline)} -> ${short(change.head)}). Inspect the isolated evidence snapshot directly.`;
+                }
+                if (change.baseline === LOCAL_CONTENT_BASELINE) {
+                    return `${heading}: files changed since the last documentation sync (compared by content; this folder has no Git history).\nChanged files:\n${fileList(change.changedFiles)}\nInspect the current files directly.`;
+                }
                 return withUncommitted(change.changedFiles.length > 0
                     ? `${heading}: changed since the last documentation sync (${short(change.baseline)} -> ${short(change.head)}).\nCommitted changes:\n${fileList(change.changedFiles)}\nInspect details with \`git -C ${change.path} diff ${short(change.baseline)}..HEAD -- <file>\`.`
                     : `${heading}: the working tree changed since the last documentation sync (${short(change.baseline)}).`, change.uncommittedFiles);
@@ -146,6 +343,28 @@ export function formatSourceChanges(changes) {
     });
     return `Source changes since the last documentation sync:\n\n${sections.join('\n\n')}`;
 }
+function formatApiDelta(diff) {
+    const lines = ['Structural API delta:'];
+    for (const item of diff.operations.added)
+        lines.push(`- operation added: ${item}`);
+    for (const item of diff.operations.removed)
+        lines.push(`- operation removed: ${item}`);
+    for (const item of diff.operations.changed)
+        lines.push(`- operation changed: ${item.id} (${item.facets.join(', ')})`);
+    for (const item of diff.schemas.added)
+        lines.push(`- schema added: ${item}`);
+    for (const item of diff.schemas.removed)
+        lines.push(`- schema removed: ${item}`);
+    for (const item of diff.schemas.changed)
+        lines.push(`- schema changed: ${item.id}`);
+    for (const item of diff.securitySchemes.added)
+        lines.push(`- authentication added: ${item}`);
+    for (const item of diff.securitySchemes.removed)
+        lines.push(`- authentication removed: ${item}`);
+    for (const item of diff.securitySchemes.changed)
+        lines.push(`- authentication changed: ${item.id}`);
+    return lines.length === 1 ? `${lines[0]} no public structural changes detected.` : lines.join('\n');
+}
 function withUncommitted(text, uncommittedFiles) {
     if (uncommittedFiles.length === 0)
         return text;
@@ -183,14 +402,26 @@ async function uncommittedChanges(path) {
     return lines(output).map((line) => line.trim());
 }
 async function sourceContentFingerprint(path) {
+    return (await localSourceManifest(path)).fingerprint;
+}
+/**
+ * Every file that counts as source content, with its digest. Git repositories
+ * contribute tracked and untracked-but-not-ignored files; a plain folder is
+ * walked directly with the same exclusions a repository would apply by
+ * convention. Secrets are never read.
+ */
+export async function localSourceManifest(path) {
     const output = await git(path, [
         'ls-files',
         '-co',
         '--exclude-standard',
         '-z',
     ]);
-    const files = (output ?? '').split('\0').filter(Boolean).sort();
+    const files = output === undefined
+        ? await walkPlainFolder(path)
+        : output.split('\0').filter(Boolean).sort();
     const hash = createHash('sha256');
+    const manifest = {};
     for (const file of files) {
         if (isSensitiveSourcePath(file))
             continue;
@@ -215,8 +446,34 @@ async function sourceContentFingerprint(path) {
         if (content !== undefined)
             hash.update(content);
         hash.update('\0');
+        manifest[file] = createHash('sha256').update(content ?? '').digest('hex');
     }
-    return hash.digest('hex');
+    return { files: manifest, fingerprint: hash.digest('hex') };
+}
+const PLAIN_FOLDER_IGNORED = new Set(['.git', 'node_modules', '.doxloop-sources', '.doxloop']);
+async function walkPlainFolder(root) {
+    const files = [];
+    const visit = async (directory) => {
+        let entries;
+        try {
+            entries = await readdir(directory, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            if (PLAIN_FOLDER_IGNORED.has(entry.name))
+                continue;
+            const absolute = join(directory, entry.name);
+            if (entry.isDirectory())
+                await visit(absolute);
+            else if (entry.isFile() || entry.isSymbolicLink()) {
+                files.push(relative(root, absolute).split('\\').join('/'));
+            }
+        }
+    };
+    await visit(root);
+    return files.sort();
 }
 function isSensitiveSourcePath(path) {
     const segments = path.toLowerCase().split(/[\\/]/);
@@ -262,6 +519,15 @@ function isSourceRecords(value) {
             (record.contentFingerprint ===
                 undefined ||
                 typeof record
-                    .contentFingerprint === 'string')));
+                    .contentFingerprint === 'string') &&
+            isFileManifest(record.files)));
+}
+function isFileManifest(value) {
+    if (value === undefined)
+        return true;
+    return (value !== null &&
+        typeof value === 'object' &&
+        !Array.isArray(value) &&
+        Object.values(value).every((digest) => typeof digest === 'string'));
 }
 //# sourceMappingURL=sync.js.map

package/dist/text-diff.d.ts

@@ -0,0 +1,9 @@
+import type { SyncChangeHunk } from './types.js';
+/**
+ * Line diffing shared by the proposal pipeline and the history index. It lives
+ * on its own so history can measure an agent's edits without importing the
+ * synchronization module that already records into history.
+ */
+export declare function textLines(text: string): string[];
+export declare function lineHunks(before: string, after: string, prefix: string): SyncChangeHunk[];
+//# sourceMappingURL=text-diff.d.ts.map

package/dist/text-diff.js

@@ -0,0 +1,59 @@
+/**
+ * Line diffing shared by the proposal pipeline and the history index. It lives
+ * on its own so history can measure an agent's edits without importing the
+ * synchronization module that already records into history.
+ */
+export function textLines(text) {
+    if (text === '')
+        return [];
+    const normalized = text.replace(/\r\n/g, '\n');
+    return normalized.endsWith('\n') ? normalized.slice(0, -1).split('\n') : normalized.split('\n');
+}
+export function lineHunks(before, after, prefix) {
+    if (before === after)
+        return [];
+    const oldLines = textLines(before);
+    const newLines = textLines(after);
+    if (oldLines.length * newLines.length > 2_000_000) {
+        return [{ id: `${prefix}-1`, oldStart: 0, oldLines, newStart: 0, newLines }];
+    }
+    const table = Array.from({ length: oldLines.length + 1 }, () => new Uint32Array(newLines.length + 1));
+    for (let old = oldLines.length - 1; old >= 0; old -= 1) {
+        for (let next = newLines.length - 1; next >= 0; next -= 1) {
+            table[old][next] = oldLines[old] === newLines[next]
+                ? table[old + 1][next + 1] + 1
+                : Math.max(table[old + 1][next], table[old][next + 1]);
+        }
+    }
+    const hunks = [];
+    let old = 0;
+    let next = 0;
+    let current;
+    const flush = () => {
+        if (!current)
+            return;
+        current.id = `${prefix}-${hunks.length + 1}`;
+        hunks.push(current);
+        current = undefined;
+    };
+    while (old < oldLines.length || next < newLines.length) {
+        if (old < oldLines.length && next < newLines.length && oldLines[old] === newLines[next]) {
+            flush();
+            old += 1;
+            next += 1;
+            continue;
+        }
+        current ??= { id: '', oldStart: old, oldLines: [], newStart: next, newLines: [] };
+        if (next < newLines.length && (old >= oldLines.length || table[old][next + 1] >= table[old + 1][next])) {
+            current.newLines.push(newLines[next]);
+            next += 1;
+        }
+        else if (old < oldLines.length) {
+            current.oldLines.push(oldLines[old]);
+            old += 1;
+        }
+    }
+    flush();
+    return hunks;
+}
+//# sourceMappingURL=text-diff.js.map