@hardlydifficult/repo-processor 1.0.0

package/dist/RepoProcessor.d.ts

@@ -0,0 +1,26 @@
+import { type GitHubClient } from "@hardlydifficult/github";
+import type { ProcessingResult, ProcessorCallbacks, ProcessorStore, ProgressCallback } from "./types.js";
+export interface RepoProcessorConfig {
+    githubClient: GitHubClient;
+    store: ProcessorStore;
+    callbacks: ProcessorCallbacks;
+    concurrency?: number;
+    branch?: string;
+}
+/**
+ * Generic pipeline for incrementally processing a GitHub repo's file tree.
+ *
+ * Pipeline: init → fetch tree → filter → diff → process changed files →
+ * remove deleted files → resolve stale dirs → process dirs bottom-up → commit.
+ */
+export declare class RepoProcessor {
+    private readonly github;
+    private readonly store;
+    private readonly callbacks;
+    private readonly concurrency;
+    private readonly branch;
+    constructor(config: RepoProcessorConfig);
+    run(owner: string, repo: string, onProgress?: ProgressCallback): Promise<ProcessingResult>;
+    private buildDirectoryContext;
+}
+//# sourceMappingURL=RepoProcessor.d.ts.map

package/dist/RepoProcessor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"RepoProcessor.d.ts","sourceRoot":"","sources":["../src/RepoProcessor.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,YAAY,EAElB,MAAM,yBAAyB,CAAC;AAIjC,OAAO,KAAK,EAIV,gBAAgB,EAChB,kBAAkB,EAClB,cAAc,EACd,gBAAgB,EACjB,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,mBAAmB;IAClC,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,EAAE,cAAc,CAAC;IACtB,SAAS,EAAE,kBAAkB,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;GAKG;AACH,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiB;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAqB;IAC/C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;gBAEpB,MAAM,EAAE,mBAAmB;IAQjC,GAAG,CACP,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,UAAU,CAAC,EAAE,gBAAgB,GAC5B,OAAO,CAAC,gBAAgB,CAAC;IA6N5B,OAAO,CAAC,qBAAqB;CAqC9B"}

package/dist/RepoProcessor.js

@@ -0,0 +1,178 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RepoProcessor = void 0;
+const collections_1 = require("@hardlydifficult/collections");
+const github_1 = require("@hardlydifficult/github");
+const text_1 = require("@hardlydifficult/text");
+const resolveDirectories_js_1 = require("./resolveDirectories.js");
+/**
+ * Generic pipeline for incrementally processing a GitHub repo's file tree.
+ *
+ * Pipeline: init → fetch tree → filter → diff → process changed files →
+ * remove deleted files → resolve stale dirs → process dirs bottom-up → commit.
+ */
+class RepoProcessor {
+    github;
+    store;
+    callbacks;
+    concurrency;
+    branch;
+    constructor(config) {
+        this.github = config.githubClient;
+        this.store = config.store;
+        this.callbacks = config.callbacks;
+        this.concurrency = config.concurrency ?? 5;
+        this.branch = config.branch ?? "main";
+    }
+    async run(owner, repo, onProgress) {
+        const emitProgress = (phase, message, filesTotal, filesCompleted, dirsTotal, dirsCompleted) => {
+            onProgress?.({
+                phase,
+                message,
+                filesTotal,
+                filesCompleted,
+                dirsTotal,
+                dirsCompleted,
+            });
+        };
+        // 1. Init store
+        emitProgress("loading", "Initializing store...", 0, 0, 0, 0);
+        await this.store.ensureReady?.(owner, repo);
+        // 2. Fetch tree
+        emitProgress("loading", "Fetching file tree...", 0, 0, 0, 0);
+        const { entries, rootSha } = await this.github
+            .repo(owner, repo)
+            .getFileTree();
+        const tree = [
+            ...entries,
+            { path: "", type: "tree", sha: rootSha },
+        ];
+        // 3. Filter blobs
+        const blobs = entries.filter((e) => e.type === "blob" && this.callbacks.shouldProcess(e));
+        // 4. Diff
+        const currentManifest = await this.store.getFileManifest(owner, repo);
+        const diff = (0, github_1.diffTree)(blobs, currentManifest);
+        const filesTotal = diff.changedFiles.length;
+        let filesCompleted = 0;
+        emitProgress("files", `Processing ${String(filesTotal)} files...`, filesTotal, 0, diff.staleDirs.length, 0);
+        // 5. Process changed files in batches
+        const fileBatches = (0, collections_1.chunk)(diff.changedFiles, this.concurrency);
+        for (const batch of fileBatches) {
+            const results = await Promise.allSettled(batch.map(async (entry) => {
+                const content = await this.github
+                    .repo(owner, repo)
+                    .getFileContent(entry.path);
+                const result = await this.callbacks.processFile({ entry, content });
+                await this.store.writeFileResult(owner, repo, entry.path, entry.sha, result);
+            }));
+            const failures = [];
+            let succeededCount = 0;
+            for (let i = 0; i < results.length; i++) {
+                const result = results[i];
+                filesCompleted++;
+                if (result.status === "rejected") {
+                    failures.push({
+                        path: batch[i]?.path ?? `unknown-${String(i)}`,
+                        reason: result.reason,
+                    });
+                }
+                else {
+                    succeededCount++;
+                }
+                emitProgress("files", `Files: ${String(filesCompleted)}/${String(filesTotal)}`, filesTotal, filesCompleted, diff.staleDirs.length, 0);
+            }
+            if (failures.length > 0) {
+                const details = failures
+                    .map((f) => `  ${f.path}: ${(0, text_1.getErrorMessage)(f.reason)}`)
+                    .join("\n");
+                throw new Error(`${String(failures.length)} file(s) failed to process:\n${details}`);
+            }
+            if (succeededCount > 0) {
+                await this.store.commitBatch(owner, repo, succeededCount);
+            }
+        }
+        // 6. Remove deleted files
+        for (const removedPath of diff.removedFiles) {
+            await this.store.deleteFileResult(owner, repo, removedPath);
+        }
+        if (diff.removedFiles.length > 0) {
+            await this.store.commitBatch(owner, repo, diff.removedFiles.length);
+        }
+        // 7. Resolve stale directories
+        const allFilePaths = blobs.map((b) => b.path);
+        const allDirs = await (0, resolveDirectories_js_1.resolveStaleDirectories)(owner, repo, diff.staleDirs, allFilePaths, tree, this.store);
+        let dirsCompleted = 0;
+        if (allDirs.length > 0) {
+            emitProgress("directories", `Processing ${String(allDirs.length)} directories...`, filesTotal, filesCompleted, allDirs.length, 0);
+            // 8. Process directories bottom-up by depth
+            const depthGroups = (0, collections_1.groupByDepth)(allDirs);
+            for (const { paths: dirsAtDepth } of depthGroups) {
+                const batches = (0, collections_1.chunk)(dirsAtDepth, this.concurrency);
+                let dirsInDepthGroup = 0;
+                for (const batch of batches) {
+                    const results = await Promise.allSettled(batch.map(async (dirPath) => {
+                        const ctx = this.buildDirectoryContext(dirPath, tree, allFilePaths);
+                        const result = await this.callbacks.processDirectory(ctx);
+                        await this.store.writeDirResult(owner, repo, dirPath, ctx.sha, result);
+                    }));
+                    const failures = [];
+                    for (let i = 0; i < results.length; i++) {
+                        const result = results[i];
+                        dirsCompleted++;
+                        dirsInDepthGroup++;
+                        if (result.status === "rejected") {
+                            failures.push({
+                                path: batch[i] ?? `unknown-${String(i)}`,
+                                reason: result.reason,
+                            });
+                        }
+                        emitProgress("directories", `Dirs: ${String(dirsCompleted)}/${String(allDirs.length)}`, filesTotal, filesCompleted, allDirs.length, dirsCompleted);
+                    }
+                    if (failures.length > 0) {
+                        const details = failures
+                            .map((f) => `  ${f.path}: ${(0, text_1.getErrorMessage)(f.reason)}`)
+                            .join("\n");
+                        throw new Error(`${String(failures.length)} directory(ies) failed to process:\n${details}`);
+                    }
+                }
+                if (dirsInDepthGroup > 0) {
+                    await this.store.commitBatch(owner, repo, dirsInDepthGroup);
+                }
+            }
+        }
+        // 9. Final safety commit
+        emitProgress("committing", "Final commit...", filesTotal, filesCompleted, allDirs.length, dirsCompleted);
+        await this.store.commitBatch(owner, repo, 0);
+        return {
+            filesProcessed: filesCompleted,
+            filesRemoved: diff.removedFiles.length,
+            dirsProcessed: dirsCompleted,
+        };
+    }
+    buildDirectoryContext(dirPath, tree, allFilePaths) {
+        const dirTreeEntry = tree.find((e) => e.type === "tree" && e.path === dirPath);
+        const sha = dirTreeEntry?.sha ?? "";
+        const prefix = dirPath === "" ? "" : `${dirPath}/`;
+        const subtreeFilePaths = allFilePaths.filter((fp) => dirPath === "" || fp.startsWith(prefix));
+        // Build immediate children list
+        const seen = new Set();
+        const children = [];
+        for (const fp of subtreeFilePaths) {
+            const relative = prefix ? fp.slice(prefix.length) : fp;
+            const slashIndex = relative.indexOf("/");
+            const isDir = slashIndex !== -1;
+            const childName = isDir ? relative.slice(0, slashIndex) : relative;
+            if (!seen.has(childName)) {
+                seen.add(childName);
+                children.push({
+                    name: childName,
+                    isDir,
+                    fullPath: dirPath === "" ? childName : `${dirPath}/${childName}`,
+                });
+            }
+        }
+        return { path: dirPath, sha, subtreeFilePaths, children, tree };
+    }
+}
+exports.RepoProcessor = RepoProcessor;
+//# sourceMappingURL=RepoProcessor.js.map

package/dist/RepoProcessor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RepoProcessor.js","sourceRoot":"","sources":["../src/RepoProcessor.ts"],"names":[],"mappings":";;;AAAA,8DAAmE;AACnE,oDAIiC;AACjC,gDAAwD;AAExD,mEAAkE;AAmBlE;;;;;GAKG;AACH,MAAa,aAAa;IACP,MAAM,CAAe;IACrB,KAAK,CAAiB;IACtB,SAAS,CAAqB;IAC9B,WAAW,CAAS;IACpB,MAAM,CAAS;IAEhC,YAAY,MAA2B;QACrC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC;QAClC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QAC1B,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QAClC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,GAAG,CACP,KAAa,EACb,IAAY,EACZ,UAA6B;QAE7B,MAAM,YAAY,GAAG,CACnB,KAAkC,EAClC,OAAe,EACf,UAAkB,EAClB,cAAsB,EACtB,SAAiB,EACjB,aAAqB,EACf,EAAE;YACR,UAAU,EAAE,CAAC;gBACX,KAAK;gBACL,OAAO;gBACP,UAAU;gBACV,cAAc;gBACd,SAAS;gBACT,aAAa;aACd,CAAC,CAAC;QACL,CAAC,CAAC;QAEF,gBAAgB;QAChB,YAAY,CAAC,SAAS,EAAE,uBAAuB,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7D,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAE5C,gBAAgB;QAChB,YAAY,CAAC,SAAS,EAAE,uBAAuB,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7D,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM;aAC3C,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC;aACjB,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,GAAyB;YACjC,GAAG,OAAO;YACV,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE;SACzC,CAAC;QAEF,kBAAkB;QAClB,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAC1B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,CAC5D,CAAC;QAEF,UAAU;QACV,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QACtE,MAAM,IAAI,GAAG,IAAA,iBAAQ,EAAC,KAAK,EAAE,eAAe,CAAC,CAAC;QAE9C,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;QAC5C,IAAI,cAAc,GAAG,CAAC,CAAC;QAEvB,YAAY,CACV,OAAO,EACP,cAAc,MAAM,CAAC,UAAU,CAAC,WAAW,EAC3C,UAAU,EACV,CAAC,EACD,IAAI,CAAC,SAAS,CAAC,MAAM,EACrB,CAAC,CACF,CAAC;QAEF,sCAAsC;QACtC,MAAM,WAAW,GAAG,IAAA,mBAAK,EAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAC/D,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;gBACxB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM;qBAC9B,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC;qBACjB,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC9B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;gBACpE,MAAM,IAAI,CAAC,KAAK,CAAC,eAAe,CAC9B,KAAK,EACL,IAAI,EACJ,KAAK,CAAC,IAAI,EACV,KAAK,CAAC,GAAG,EACT,MAAM,CACP,CAAC;YACJ,CAAC,CAAC,CACH,CAAC;YAEF,MAAM,QAAQ,GAAwC,EAAE,CAAC;YACzD,IAAI,cAAc,GAAG,CAAC,CAAC;YACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACxC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;gBAC1B,cAAc,EAAE,CAAC;gBACjB,IAAI,MAAM,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;oBACjC,QAAQ,CAAC,IAAI,CAAC;wBACZ,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,WAAW,MAAM,CAAC,CAAC,CAAC,EAAE;wBAC9C,MAAM,EAAE,MAAM,CAAC,MAAM;qBACtB,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,cAAc,EAAE,CAAC;gBACnB,CAAC;gBACD,YAAY,CACV,OAAO,EACP,UAAU,MAAM,CAAC,cAAc,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,EAAE,EACxD,UAAU,EACV,cAAc,EACd,IAAI,CAAC,SAAS,CAAC,MAAM,EACrB,CAAC,CACF,CAAC;YACJ,CAAC;YAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,OAAO,GAAG,QAAQ;qBACrB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,IAAA,sBAAe,EAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;qBACvD,IAAI,CAAC,IAAI,CAAC,CAAC;gBACd,MAAM,IAAI,KAAK,CACb,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,gCAAgC,OAAO,EAAE,CACpE,CAAC;YACJ,CAAC;YAED,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC;QAED,0BAA0B;QAC1B,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YAC5C,MAAM,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,KAAK,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACtE,CAAC;QAED,+BAA+B;QAC/B,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,MAAM,IAAA,+CAAuB,EAC3C,KAAK,EACL,IAAI,EACJ,IAAI,CAAC,SAAS,EACd,YAAY,EACZ,IAAI,EACJ,IAAI,CAAC,KAAK,CACX,CAAC;QACF,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,YAAY,CACV,aAAa,EACb,cAAc,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,iBAAiB,EACrD,UAAU,EACV,cAAc,EACd,OAAO,CAAC,MAAM,EACd,CAAC,CACF,CAAC;YAEF,4CAA4C;YAC5C,MAAM,WAAW,GAAG,IAAA,0BAAY,EAAC,OAAO,CAAC,CAAC;YAE1C,KAAK,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,WAAW,EAAE,CAAC;gBACjD,MAAM,OAAO,GAAG,IAAA,mBAAK,EAAC,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBACrD,IAAI,gBAAgB,GAAG,CAAC,CAAC;gBAEzB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;oBAC5B,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;wBAC1B,MAAM,GAAG,GAAG,IAAI,CAAC,qBAAqB,CACpC,OAAO,EACP,IAAI,EACJ,YAAY,CACb,CAAC;wBACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;wBAC1D,MAAM,IAAI,CAAC,KAAK,CAAC,cAAc,CAC7B,KAAK,EACL,IAAI,EACJ,OAAO,EACP,GAAG,CAAC,GAAG,EACP,MAAM,CACP,CAAC;oBACJ,CAAC,CAAC,CACH,CAAC;oBAEF,MAAM,QAAQ,GAAwC,EAAE,CAAC;oBACzD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;wBACxC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;wBAC1B,aAAa,EAAE,CAAC;wBAChB,gBAAgB,EAAE,CAAC;wBACnB,IAAI,MAAM,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;4BACjC,QAAQ,CAAC,IAAI,CAAC;gCACZ,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,WAAW,MAAM,CAAC,CAAC,CAAC,EAAE;gCACxC,MAAM,EAAE,MAAM,CAAC,MAAM;6BACtB,CAAC,CAAC;wBACL,CAAC;wBACD,YAAY,CACV,aAAa,EACb,SAAS,MAAM,CAAC,aAAa,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAC1D,UAAU,EACV,cAAc,EACd,OAAO,CAAC,MAAM,EACd,aAAa,CACd,CAAC;oBACJ,CAAC;oBAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACxB,MAAM,OAAO,GAAG,QAAQ;6BACrB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,IAAA,sBAAe,EAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;6BACvD,IAAI,CAAC,IAAI,CAAC,CAAC;wBACd,MAAM,IAAI,KAAK,CACb,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,uCAAuC,OAAO,EAAE,CAC3E,CAAC;oBACJ,CAAC;gBACH,CAAC;gBAED,IAAI,gBAAgB,GAAG,CAAC,EAAE,CAAC;oBACzB,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,EAAE,gBAAgB,CAAC,CAAC;gBAC9D,CAAC;YACH,CAAC;QACH,CAAC;QAED,yBAAyB;QACzB,YAAY,CACV,YAAY,EACZ,iBAAiB,EACjB,UAAU,EACV,cAAc,EACd,OAAO,CAAC,MAAM,EACd,aAAa,CACd,CAAC;QACF,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAE7C,OAAO;YACL,cAAc,EAAE,cAAc;YAC9B,YAAY,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM;YACtC,aAAa,EAAE,aAAa;SAC7B,CAAC;IACJ,CAAC;IAEO,qBAAqB,CAC3B,OAAe,EACf,IAA0B,EAC1B,YAA+B;QAE/B,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAC5B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,CAC/C,CAAC;QACF,MAAM,GAAG,GAAG,YAAY,EAAE,GAAG,IAAI,EAAE,CAAC;QAEpC,MAAM,MAAM,GAAG,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,OAAO,GAAG,CAAC;QACnD,MAAM,gBAAgB,GAAG,YAAY,CAAC,MAAM,CAC1C,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,KAAK,EAAE,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,CAChD,CAAC;QAEF,gCAAgC;QAChC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,MAAM,QAAQ,GAAqB,EAAE,CAAC;QAEtC,KAAK,MAAM,EAAE,IAAI,gBAAgB,EAAE,CAAC;YAClC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACvD,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACzC,MAAM,KAAK,GAAG,UAAU,KAAK,CAAC,CAAC,CAAC;YAChC,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;YAEnE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBACzB,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBACpB,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,SAAS;oBACf,KAAK;oBACL,QAAQ,EAAE,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,OAAO,IAAI,SAAS,EAAE;iBACjE,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,gBAAgB,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAClE,CAAC;CACF;AArRD,sCAqRC"}

package/dist/index.d.ts

@@ -0,0 +1,5 @@
+export { RepoProcessor } from "./RepoProcessor.js";
+export type { RepoProcessorConfig } from "./RepoProcessor.js";
+export { resolveStaleDirectories } from "./resolveDirectories.js";
+export type { ProcessorStore, ProcessorCallbacks, FileContext, DirectoryContext, DirectoryChild, ProcessingProgress, ProgressCallback, ProcessingResult, } from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,YAAY,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAClE,YAAY,EACV,cAAc,EACd,kBAAkB,EAClB,WAAW,EACX,gBAAgB,EAChB,cAAc,EACd,kBAAkB,EAClB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,YAAY,CAAC"}

package/dist/index.js

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveStaleDirectories = exports.RepoProcessor = void 0;
+var RepoProcessor_js_1 = require("./RepoProcessor.js");
+Object.defineProperty(exports, "RepoProcessor", { enumerable: true, get: function () { return RepoProcessor_js_1.RepoProcessor; } });
+var resolveDirectories_js_1 = require("./resolveDirectories.js");
+Object.defineProperty(exports, "resolveStaleDirectories", { enumerable: true, get: function () { return resolveDirectories_js_1.resolveStaleDirectories; } });
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,uDAAmD;AAA1C,iHAAA,aAAa,OAAA;AAEtB,iEAAkE;AAAzD,gIAAA,uBAAuB,OAAA"}

package/dist/resolveDirectories.d.ts

@@ -0,0 +1,14 @@
+import type { TreeEntry } from "@hardlydifficult/github";
+import type { ProcessorStore } from "./types.js";
+/**
+ * Find all directories that need (re-)processing.
+ *
+ * Combines two sources:
+ * 1. Stale dirs from diffTree (directories with changed/removed children)
+ * 2. Any directory whose stored SHA is missing or differs from the current tree SHA
+ *
+ * The second source handles recovery after partial failures and catches
+ * directories whose tree SHA changed without any processable file changes.
+ */
+export declare function resolveStaleDirectories(owner: string, repo: string, staleDirsFromDiff: readonly string[], allFilePaths: readonly string[], tree: readonly TreeEntry[], store: ProcessorStore): Promise<string[]>;
+//# sourceMappingURL=resolveDirectories.d.ts.map

package/dist/resolveDirectories.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolveDirectories.d.ts","sourceRoot":"","sources":["../src/resolveDirectories.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEjD;;;;;;;;;GASG;AACH,wBAAsB,uBAAuB,CAC3C,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,iBAAiB,EAAE,SAAS,MAAM,EAAE,EACpC,YAAY,EAAE,SAAS,MAAM,EAAE,EAC/B,IAAI,EAAE,SAAS,SAAS,EAAE,EAC1B,KAAK,EAAE,cAAc,GACpB,OAAO,CAAC,MAAM,EAAE,CAAC,CAyCnB"}

package/dist/resolveDirectories.js

@@ -0,0 +1,52 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveStaleDirectories = resolveStaleDirectories;
+/**
+ * Find all directories that need (re-)processing.
+ *
+ * Combines two sources:
+ * 1. Stale dirs from diffTree (directories with changed/removed children)
+ * 2. Any directory whose stored SHA is missing or differs from the current tree SHA
+ *
+ * The second source handles recovery after partial failures and catches
+ * directories whose tree SHA changed without any processable file changes.
+ */
+async function resolveStaleDirectories(owner, repo, staleDirsFromDiff, allFilePaths, tree, store) {
+    // Build map of directory path → current tree SHA
+    const treeShaByDir = new Map();
+    for (const entry of tree) {
+        if (entry.type === "tree") {
+            treeShaByDir.set(entry.path, entry.sha);
+        }
+    }
+    // Collect all directories that should be processed (derived from file paths + root)
+    const allExpectedDirs = new Set();
+    for (const filePath of allFilePaths) {
+        const parts = filePath.split("/");
+        for (let i = 1; i < parts.length; i++) {
+            allExpectedDirs.add(parts.slice(0, i).join("/"));
+        }
+    }
+    // Always include root
+    allExpectedDirs.add("");
+    // Start with stale dirs from diff
+    const needed = new Set(staleDirsFromDiff);
+    // Check every expected directory for a missing or stale stored SHA
+    for (const dirPath of allExpectedDirs) {
+        if (needed.has(dirPath)) {
+            continue;
+        }
+        const storedSha = await store.getDirSha(owner, repo, dirPath);
+        if (storedSha === null) {
+            needed.add(dirPath);
+        }
+        else {
+            const currentSha = treeShaByDir.get(dirPath) ?? "";
+            if (storedSha !== currentSha) {
+                needed.add(dirPath);
+            }
+        }
+    }
+    return [...needed];
+}
+//# sourceMappingURL=resolveDirectories.js.map

package/dist/resolveDirectories.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolveDirectories.js","sourceRoot":"","sources":["../src/resolveDirectories.ts"],"names":[],"mappings":";;AAcA,0DAgDC;AA1DD;;;;;;;;;GASG;AACI,KAAK,UAAU,uBAAuB,CAC3C,KAAa,EACb,IAAY,EACZ,iBAAoC,EACpC,YAA+B,EAC/B,IAA0B,EAC1B,KAAqB;IAErB,iDAAiD;IACjD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC/C,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;QACzB,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC1B,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,oFAAoF;IACpF,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU,CAAC;IAC1C,KAAK,MAAM,QAAQ,IAAI,YAAY,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IACD,sBAAsB;IACtB,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAExB,kCAAkC;IAClC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,iBAAiB,CAAC,CAAC;IAE1C,mEAAmE;IACnE,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;QACtC,IAAI,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACxB,SAAS;QACX,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QAC9D,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACvB,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;aAAM,CAAC;YACN,MAAM,UAAU,GAAG,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YACnD,IAAI,SAAS,KAAK,UAAU,EAAE,CAAC;gBAC7B,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC;AACrB,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,66 @@
+import type { FileManifest, TreeEntry } from "@hardlydifficult/github";
+export type { TreeEntry, FileManifest };
+/** Manifest of previously processed file SHAs (path → blob SHA). */
+export type { FileManifest as ProcessorFileManifest };
+/** Consumer-implemented persistence layer. */
+export interface ProcessorStore {
+    /** One-time init (e.g. ensure local clone). Optional. */
+    ensureReady?(owner: string, repo: string): Promise<void>;
+    /** Return manifest of previously processed file SHAs (path → blob SHA). */
+    getFileManifest(owner: string, repo: string): Promise<FileManifest>;
+    /** Return stored SHA for a directory. Null if not stored. */
+    getDirSha(owner: string, repo: string, dirPath: string): Promise<string | null>;
+    /** Persist result for a processed file. */
+    writeFileResult(owner: string, repo: string, path: string, sha: string, result: unknown): Promise<void>;
+    /** Persist result for a processed directory. */
+    writeDirResult(owner: string, repo: string, path: string, sha: string, result: unknown): Promise<void>;
+    /** Delete stored result for a removed file. */
+    deleteFileResult(owner: string, repo: string, path: string): Promise<void>;
+    /** Commit current batch of changes. */
+    commitBatch(owner: string, repo: string, count: number): Promise<void>;
+}
+/** Context passed to processFile callback. */
+export interface FileContext {
+    readonly entry: TreeEntry;
+    readonly content: string;
+}
+/** An immediate child of a directory being processed. */
+export interface DirectoryChild {
+    readonly name: string;
+    readonly isDir: boolean;
+    readonly fullPath: string;
+}
+/** Context passed to processDirectory callback. */
+export interface DirectoryContext {
+    readonly path: string;
+    readonly sha: string;
+    readonly subtreeFilePaths: readonly string[];
+    readonly children: readonly DirectoryChild[];
+    readonly tree: readonly TreeEntry[];
+}
+/** Consumer-provided domain logic. */
+export interface ProcessorCallbacks {
+    /** Filter: which tree entries should be processed? */
+    shouldProcess(entry: TreeEntry): boolean;
+    /** Process a single changed file. Return value passed to store.writeFileResult. */
+    processFile(ctx: FileContext): Promise<unknown>;
+    /** Process a directory after all children. Return value passed to store.writeDirResult. */
+    processDirectory(ctx: DirectoryContext): Promise<unknown>;
+}
+/** Progress reported during a run. */
+export interface ProcessingProgress {
+    phase: "loading" | "files" | "directories" | "committing";
+    message: string;
+    filesTotal: number;
+    filesCompleted: number;
+    dirsTotal: number;
+    dirsCompleted: number;
+}
+export type ProgressCallback = (progress: ProcessingProgress) => void;
+/** Result returned by RepoProcessor.run(). */
+export interface ProcessingResult {
+    filesProcessed: number;
+    filesRemoved: number;
+    dirsProcessed: number;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAEvE,YAAY,EAAE,SAAS,EAAE,YAAY,EAAE,CAAC;AAExC,oEAAoE;AACpE,YAAY,EAAE,YAAY,IAAI,qBAAqB,EAAE,CAAC;AAEtD,8CAA8C;AAC9C,MAAM,WAAW,cAAc;IAC7B,yDAAyD;IACzD,WAAW,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,2EAA2E;IAC3E,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACpE,6DAA6D;IAC7D,SAAS,CACP,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC1B,2CAA2C;IAC3C,eAAe,CACb,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,OAAO,GACd,OAAO,CAAC,IAAI,CAAC,CAAC;IACjB,gDAAgD;IAChD,cAAc,CACZ,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,OAAO,GACd,OAAO,CAAC,IAAI,CAAC,CAAC;IACjB,+CAA+C;IAC/C,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3E,uCAAuC;IACvC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACxE;AAED,8CAA8C;AAC9C,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,yDAAyD;AACzD,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,mDAAmD;AACnD,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,gBAAgB,EAAE,SAAS,MAAM,EAAE,CAAC;IAC7C,QAAQ,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAC;IAC7C,QAAQ,CAAC,IAAI,EAAE,SAAS,SAAS,EAAE,CAAC;CACrC;AAED,sCAAsC;AACtC,MAAM,WAAW,kBAAkB;IACjC,sDAAsD;IACtD,aAAa,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC;IACzC,mFAAmF;IACnF,WAAW,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAChD,2FAA2F;IAC3F,gBAAgB,CAAC,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC3D;AAED,sCAAsC;AACtC,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,SAAS,GAAG,OAAO,GAAG,aAAa,GAAG,YAAY,CAAC;IAC1D,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,gBAAgB,GAAG,CAAC,QAAQ,EAAE,kBAAkB,KAAK,IAAI,CAAC;AAEtE,8CAA8C;AAC9C,MAAM,WAAW,gBAAgB;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACvB"}

package/dist/types.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@hardlydifficult/repo-processor",
+  "version": "1.0.0",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
+    "lint": "tsc --noEmit",
+    "clean": "rm -rf dist"
+  },
+  "dependencies": {
+    "@hardlydifficult/collections": "1.0.0",
+    "@hardlydifficult/github": "1.0.22",
+    "@hardlydifficult/text": "1.0.15"
+  },
+  "devDependencies": {
+    "@types/node": "25.2.3",
+    "typescript": "5.9.3",
+    "vitest": "4.0.18"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}