@tomingtoming/kioq 0.7.0

package/dist/src/storage/githubStorage.js

@@ -0,0 +1,447 @@
+import { execFile } from "node:child_process";
+import { promises as fs } from "node:fs";
+import path from "node:path";
+import { promisify } from "node:util";
+import { LocalFsStorage } from "./localFsStorage.js";
+const execFileAsync = promisify(execFile);
+const MAX_RETRIES = 3;
+const RETRY_BASE_MS = 200;
+class GitHubApiError extends Error {
+    status;
+    endpoint;
+    detail;
+    retryable;
+    retryAfterMs;
+    constructor(status, endpoint, detail, retryable, retryAfterMs) {
+        super(`GitHub API error (${status}) on ${endpoint}: ${detail}`);
+        this.status = status;
+        this.endpoint = endpoint;
+        this.detail = detail;
+        this.retryable = retryable;
+        this.retryAfterMs = retryAfterMs;
+        this.name = "GitHubApiError";
+    }
+}
+class RefUpdateConflictError extends Error {
+    constructor() {
+        super("GitHub ref update conflict: ref was updated concurrently");
+        this.name = "RefUpdateConflictError";
+    }
+}
+export class GitHubStorage {
+    githubConfig;
+    rootPath;
+    local;
+    apiRoot;
+    apiBase;
+    branch;
+    fetchFn;
+    execFileAsyncFn;
+    setupValidated = false;
+    constructor(githubConfig, rootPath, deps = {}) {
+        this.githubConfig = githubConfig;
+        this.rootPath = rootPath;
+        this.local = new LocalFsStorage(rootPath);
+        this.apiRoot = "https://api.github.com";
+        this.apiBase = `https://api.github.com/repos/${githubConfig.owner}/${githubConfig.repo}`;
+        this.branch = githubConfig.branch;
+        this.fetchFn = deps.fetchFn ?? fetch;
+        this.execFileAsyncFn = deps.execFileAsyncFn ?? execFileAsync;
+    }
+    // --- Reads: delegate to local clone ---
+    stat(relativePath) {
+        return this.local.stat(relativePath);
+    }
+    readFile(relativePath) {
+        return this.local.readFile(relativePath);
+    }
+    readdir(relativePath) {
+        return this.local.readdir(relativePath);
+    }
+    listMarkdownFiles() {
+        return this.local.listMarkdownFiles();
+    }
+    // --- Writes: GitHub Git Data API ---
+    async writeFile(relativePath, content) {
+        await this.applyBatch([{ kind: "write", relativePath, content }]);
+    }
+    async deleteFile(relativePath) {
+        await this.applyBatch([{ kind: "delete", relativePath }]);
+    }
+    async applyBatch(operations, message) {
+        if (operations.length === 0) {
+            return;
+        }
+        const commitMessage = message ?? this.generateCommitMessage(operations);
+        for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+            try {
+                await this.commitViaApi(operations, commitMessage);
+                await this.pullLocal("GitHub write succeeded, but local clone sync failed");
+                return;
+            }
+            catch (error) {
+                if (error instanceof RefUpdateConflictError && attempt < MAX_RETRIES) {
+                    await this.waitBeforeRetry(RETRY_BASE_MS * Math.pow(2, attempt));
+                    await this.tryRefreshLocalClone();
+                    continue;
+                }
+                if (this.isRetryableBatchError(error) && attempt < MAX_RETRIES) {
+                    await this.waitBeforeRetry(this.retryDelayMs(error, attempt));
+                    await this.tryRefreshLocalClone();
+                    continue;
+                }
+                throw error;
+            }
+        }
+    }
+    // --- Lifecycle ---
+    async ensureRoot() {
+        await this.ensureLocalClone();
+        await this.validateSetup();
+        await this.pullLocal("git pull --ff-only failed during GitHubStorage startup");
+    }
+    async sync() {
+        await this.validateSetup();
+        await this.pullLocal("git pull --ff-only failed during GitHubStorage sync");
+    }
+    // --- Private: setup validation ---
+    async ensureLocalClone() {
+        const gitDir = path.join(this.rootPath, ".git");
+        try {
+            const s = await fs.stat(gitDir);
+            if (!s.isDirectory()) {
+                throw new Error(`${this.rootPath} is not a git repository`);
+            }
+        }
+        catch (error) {
+            if (error instanceof Error && error.message.includes("is not a git repository")) {
+                throw error;
+            }
+            throw new Error(`Local clone not found at ${this.rootPath}. `
+                + `Clone the repository first: git clone https://github.com/${this.githubConfig.owner}/${this.githubConfig.repo}.git ${this.rootPath}`);
+        }
+    }
+    async validateSetup() {
+        if (this.setupValidated) {
+            return;
+        }
+        const remoteUrl = await this.getOriginRemoteUrl();
+        if (!this.remoteMatchesConfiguredRepo(remoteUrl)) {
+            throw new Error(`Local clone origin mismatch: expected ${this.repoSlug()}, got ${remoteUrl}`);
+        }
+        const viewer = await this.validateToken();
+        const repo = await this.validateRepositoryAccess(viewer.login);
+        await this.validateBranch(repo.default_branch);
+        this.setupValidated = true;
+    }
+    // --- Private: Git Data API flow ---
+    async commitViaApi(operations, message) {
+        const ref = await this.apiRequest("GET", `/git/ref/heads/${this.branch}`);
+        const parentCommitSha = ref.object.sha;
+        const parentCommit = await this.apiRequest("GET", `/git/commits/${parentCommitSha}`);
+        const baseTreeSha = parentCommit.tree.sha;
+        const writes = operations.filter((op) => op.kind === "write");
+        const deletes = operations.filter((op) => op.kind === "delete");
+        const blobResults = await Promise.all(writes.map(async (op) => {
+            const blob = await this.apiRequest("POST", "/git/blobs", { content: op.content, encoding: "utf-8" });
+            return { repoPath: op.relativePath, sha: blob.sha };
+        }));
+        let newTreeSha;
+        if (deletes.length === 0) {
+            const treeEntries = blobResults.map(({ repoPath, sha }) => ({
+                path: repoPath,
+                mode: "100644",
+                type: "blob",
+                sha,
+            }));
+            const tree = await this.apiRequest("POST", "/git/trees", { base_tree: baseTreeSha, tree: treeEntries });
+            newTreeSha = tree.sha;
+        }
+        else {
+            const fullTree = await this.apiRequest("GET", `/git/trees/${baseTreeSha}?recursive=1`);
+            if (fullTree.truncated) {
+                throw new Error("Repository tree is too large for the GitHub API (truncated response)");
+            }
+            const deleteSet = new Set(deletes.map((op) => op.relativePath));
+            const writeMap = new Map(blobResults.map(({ repoPath, sha }) => [repoPath, sha]));
+            const surviving = fullTree.tree
+                .filter((entry) => entry.type === "blob")
+                .filter((entry) => !deleteSet.has(entry.path))
+                .filter((entry) => !writeMap.has(entry.path))
+                .map((entry) => ({
+                path: entry.path,
+                mode: entry.mode,
+                type: "blob",
+                sha: entry.sha,
+            }));
+            const writeEntries = blobResults.map(({ repoPath, sha }) => ({
+                path: repoPath,
+                mode: "100644",
+                type: "blob",
+                sha,
+            }));
+            const tree = await this.apiRequest("POST", "/git/trees", { tree: [...surviving, ...writeEntries] });
+            newTreeSha = tree.sha;
+        }
+        const commit = await this.apiRequest("POST", "/git/commits", {
+            message,
+            tree: newTreeSha,
+            parents: [parentCommitSha],
+        });
+        await this.apiRequest("PATCH", `/git/refs/heads/${this.branch}`, { sha: commit.sha });
+    }
+    // --- Private: helpers ---
+    generateCommitMessage(operations) {
+        if (operations.length === 1) {
+            const op = operations[0];
+            return `kioq: ${op.kind} ${op.relativePath}`;
+        }
+        const writes = operations.filter((op) => op.kind === "write").length;
+        const deletions = operations.filter((op) => op.kind === "delete").length;
+        const parts = [];
+        if (writes > 0) {
+            parts.push(`write ${writes} file${writes > 1 ? "s" : ""}`);
+        }
+        if (deletions > 0) {
+            parts.push(`delete ${deletions} file${deletions > 1 ? "s" : ""}`);
+        }
+        return `kioq: ${parts.join(", ")}`;
+    }
+    async apiRequest(method, apiPath, body, options = {}) {
+        const url = apiPath.startsWith("https://") ? apiPath : `${this.apiBase}${apiPath}`;
+        const maxAttempts = options.retryTransient ? MAX_RETRIES + 1 : 1;
+        for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
+            let response;
+            try {
+                response = await this.fetchFn(url, {
+                    method,
+                    headers: this.buildHeaders(options.useAuth ?? true),
+                    body: body ? JSON.stringify(body) : undefined,
+                });
+            }
+            catch (error) {
+                if (options.retryTransient && this.isTransientNetworkError(error) && attempt < maxAttempts - 1) {
+                    await this.waitBeforeRetry(this.retryDelayMs(error, attempt));
+                    continue;
+                }
+                throw error;
+            }
+            if (!response.ok) {
+                if (response.status === 422 && apiPath.startsWith("/git/refs/")) {
+                    throw new RefUpdateConflictError();
+                }
+                const text = await response.text();
+                const retryAfterMs = this.retryAfterMs(response.headers, text);
+                const retryable = this.isRetryableStatus(response.status, response.headers, text);
+                if (options.retryTransient && retryable && attempt < maxAttempts - 1) {
+                    await this.waitBeforeRetry(retryAfterMs ?? this.retryDelayMs(undefined, attempt));
+                    continue;
+                }
+                throw new GitHubApiError(response.status, `${method} ${apiPath}`, this.describeApiError(text), retryable, retryAfterMs);
+            }
+            return response.json();
+        }
+        throw new Error(`GitHub API request exhausted retries: ${method} ${apiPath}`);
+    }
+    buildHeaders(useAuth) {
+        const headers = {
+            "Accept": "application/vnd.github+json",
+            "X-GitHub-Api-Version": "2022-11-28",
+            "Content-Type": "application/json",
+        };
+        if (useAuth) {
+            headers.Authorization = `Bearer ${this.githubConfig.token}`;
+        }
+        return headers;
+    }
+    async validateToken() {
+        try {
+            return await this.apiRequest("GET", `${this.apiRoot}/user`, undefined, { retryTransient: true });
+        }
+        catch (error) {
+            if (error instanceof GitHubApiError) {
+                if (error.status === 401) {
+                    throw new Error("GitHub token is invalid or expired");
+                }
+                if (error.status === 403) {
+                    throw new Error(`GitHub token was rejected during authentication: ${error.detail}`);
+                }
+            }
+            throw error;
+        }
+    }
+    async validateRepositoryAccess(viewerLogin) {
+        try {
+            const repo = await this.apiRequest("GET", "", undefined, { retryTransient: true });
+            if (repo.permissions && repo.permissions.push !== true) {
+                throw new Error(`GitHub token can read ${this.repoSlug()}, but does not have push access for writes`);
+            }
+            return repo;
+        }
+        catch (error) {
+            if (error instanceof GitHubApiError && error.status === 404) {
+                const publicRepoExists = await this.publicRepositoryExists();
+                if (publicRepoExists) {
+                    throw new Error(`GitHub token cannot access ${this.repoSlug()}. Check repo access or token scopes for ${viewerLogin}`);
+                }
+                throw new Error(`GitHub repository ${this.repoSlug()} was not found, or the token cannot access it if it is private`);
+            }
+            throw error;
+        }
+    }
+    async validateBranch(defaultBranch) {
+        try {
+            return await this.apiRequest("GET", `/branches/${encodeURIComponent(this.branch)}`, undefined, { retryTransient: true });
+        }
+        catch (error) {
+            if (error instanceof GitHubApiError && error.status === 404) {
+                if (defaultBranch && defaultBranch !== this.branch) {
+                    throw new Error(`GitHub branch "${this.branch}" was not found in ${this.repoSlug()} (default branch: "${defaultBranch}")`);
+                }
+                throw new Error(`GitHub branch "${this.branch}" was not found in ${this.repoSlug()}`);
+            }
+            throw error;
+        }
+    }
+    async publicRepositoryExists() {
+        try {
+            await this.apiRequest("GET", "", undefined, {
+                useAuth: false,
+                retryTransient: true,
+            });
+            return true;
+        }
+        catch (error) {
+            if (error instanceof GitHubApiError && error.status === 404) {
+                return false;
+            }
+            return false;
+        }
+    }
+    async getOriginRemoteUrl() {
+        try {
+            const { stdout } = await this.execFileAsyncFn("git", ["remote", "get-url", "origin"], {
+                cwd: this.rootPath,
+                timeout: 15_000,
+            });
+            const remoteUrl = stdout.trim();
+            if (remoteUrl.length === 0) {
+                throw new Error("origin remote is empty");
+            }
+            return remoteUrl;
+        }
+        catch (error) {
+            throw new Error(`Failed to read git origin remote in ${this.rootPath}: ${this.normalizeExecError(error)}`);
+        }
+    }
+    remoteMatchesConfiguredRepo(remoteUrl) {
+        const normalizedRemote = remoteUrl.trim().replace(/\.git$/i, "").toLowerCase();
+        const slug = this.repoSlug().toLowerCase();
+        return normalizedRemote.endsWith(`/${slug}`) || normalizedRemote.endsWith(`:${slug}`);
+    }
+    repoSlug() {
+        return `${this.githubConfig.owner}/${this.githubConfig.repo}`;
+    }
+    async pullLocal(context) {
+        try {
+            await this.execFileAsyncFn("git", ["pull", "--ff-only"], {
+                cwd: this.rootPath,
+                timeout: 30_000,
+            });
+        }
+        catch (error) {
+            throw new Error(`${context}: ${this.normalizeExecError(error)}`);
+        }
+    }
+    async tryRefreshLocalClone() {
+        try {
+            await this.pullLocal("git pull --ff-only failed while refreshing local clone");
+        }
+        catch {
+            // best-effort only
+        }
+    }
+    isRetryableBatchError(error) {
+        return (error instanceof GitHubApiError && error.retryable)
+            || this.isTransientNetworkError(error);
+    }
+    retryDelayMs(error, attempt) {
+        if (error instanceof GitHubApiError && error.retryAfterMs !== undefined) {
+            return error.retryAfterMs;
+        }
+        return RETRY_BASE_MS * Math.pow(2, attempt);
+    }
+    isRetryableStatus(status, headers, detail) {
+        if ([408, 429, 500, 502, 503, 504].includes(status)) {
+            return true;
+        }
+        if (status === 403) {
+            const remaining = headers.get("x-ratelimit-remaining");
+            if (remaining === "0") {
+                return true;
+            }
+            const normalizedDetail = detail.toLowerCase();
+            if (normalizedDetail.includes("secondary rate limit")) {
+                return true;
+            }
+        }
+        return false;
+    }
+    retryAfterMs(headers, detail) {
+        const retryAfter = headers.get("retry-after");
+        if (retryAfter) {
+            const seconds = Number(retryAfter);
+            if (Number.isFinite(seconds) && seconds >= 0) {
+                return seconds * 1000;
+            }
+        }
+        const resetAt = headers.get("x-ratelimit-reset");
+        if (resetAt) {
+            const seconds = Number(resetAt);
+            if (Number.isFinite(seconds) && seconds > 0) {
+                const delayMs = (seconds * 1000) - Date.now();
+                if (delayMs > 0) {
+                    return delayMs;
+                }
+            }
+        }
+        if (detail.toLowerCase().includes("secondary rate limit")) {
+            return 5_000;
+        }
+        return undefined;
+    }
+    isTransientNetworkError(error) {
+        if (!(error instanceof Error)) {
+            return false;
+        }
+        const detail = `${error.name}: ${error.message}`;
+        return /fetch failed|network|timeout|timed out|econnreset|enotfound|eai_again/i.test(detail);
+    }
+    describeApiError(text) {
+        try {
+            const parsed = JSON.parse(text);
+            const parts = [parsed.message, ...(parsed.errors ?? []).map((item) => item.message)]
+                .filter((value) => typeof value === "string" && value.trim().length > 0);
+            if (parts.length > 0) {
+                return parts.join(" | ");
+            }
+        }
+        catch {
+            // fall back to raw text
+        }
+        const trimmed = text.trim();
+        return trimmed.length > 0 ? trimmed : "unknown error";
+    }
+    normalizeExecError(error) {
+        if (!(error instanceof Error)) {
+            return String(error);
+        }
+        const stdout = "stdout" in error && typeof error.stdout === "string" ? error.stdout.trim() : "";
+        const stderr = "stderr" in error && typeof error.stderr === "string" ? error.stderr.trim() : "";
+        const parts = [error.message, stderr, stdout].filter((value) => value.length > 0);
+        return parts.join(" | ");
+    }
+    async waitBeforeRetry(delayMs) {
+        await new Promise((resolve) => setTimeout(resolve, delayMs));
+    }
+}

package/dist/src/storage/index.js

@@ -0,0 +1,3 @@
+export { GitHubStorage } from "./githubStorage.js";
+export { LocalFsStorage } from "./localFsStorage.js";
+export { toPosix } from "./types.js";

package/dist/src/storage/localFsStorage.js

@@ -0,0 +1,125 @@
+import { promises as fs } from "node:fs";
+import path from "node:path";
+import { toPosix } from "./types.js";
+const IGNORED_DIRS = new Set([".git", "node_modules", ".obsidian"]);
+const MARKDOWN_EXTENSION = ".md";
+export class LocalFsStorage {
+    rootPath;
+    constructor(rootPath) {
+        this.rootPath = rootPath;
+    }
+    async stat(relativePath) {
+        try {
+            const s = await fs.stat(this.resolve(relativePath));
+            return {
+                exists: true,
+                isFile: s.isFile(),
+                isDirectory: s.isDirectory(),
+                birthtime: s.birthtime,
+                mtime: s.mtime,
+            };
+        }
+        catch {
+            return { exists: false, isFile: false, isDirectory: false };
+        }
+    }
+    async readFile(relativePath) {
+        return fs.readFile(this.resolve(relativePath), "utf8");
+    }
+    async readdir(relativePath) {
+        const entries = await fs.readdir(this.resolve(relativePath), { withFileTypes: true });
+        return entries.map((entry) => ({
+            name: entry.name,
+            isFile: entry.isFile(),
+            isDirectory: entry.isDirectory(),
+        }));
+    }
+    async listMarkdownFiles() {
+        const files = [];
+        const queue = [this.rootPath];
+        while (queue.length > 0) {
+            const currentDir = queue.pop();
+            if (!currentDir) {
+                continue;
+            }
+            let entries;
+            try {
+                entries = await fs.readdir(currentDir, { withFileTypes: true });
+            }
+            catch {
+                continue;
+            }
+            for (const entry of entries) {
+                const entryPath = path.join(currentDir, entry.name);
+                if (entry.isDirectory()) {
+                    if (IGNORED_DIRS.has(entry.name) || entry.name.startsWith(".")) {
+                        continue;
+                    }
+                    queue.push(entryPath);
+                    continue;
+                }
+                if (entry.isFile() && entry.name.endsWith(MARKDOWN_EXTENSION)) {
+                    files.push(toPosix(path.relative(this.rootPath, entryPath)));
+                }
+            }
+        }
+        return files;
+    }
+    async applyBatch(operations) {
+        for (const op of operations) {
+            switch (op.kind) {
+                case "write":
+                    await this.writeFile(op.relativePath, op.content);
+                    break;
+                case "delete":
+                    await this.deleteFile(op.relativePath);
+                    break;
+            }
+        }
+    }
+    async writeFile(relativePath, content) {
+        const absPath = this.resolve(relativePath);
+        await fs.mkdir(path.dirname(absPath), { recursive: true });
+        await fs.writeFile(absPath, content, "utf8");
+    }
+    async deleteFile(relativePath) {
+        const absPath = this.resolve(relativePath);
+        await fs.unlink(absPath);
+        await this.pruneEmptyDirectories(path.dirname(absPath));
+    }
+    async ensureRoot() {
+        await fs.mkdir(this.rootPath, { recursive: true });
+    }
+    async sync() {
+        // no-op for local storage
+    }
+    resolve(relativePath) {
+        if (relativePath === "" || relativePath === ".") {
+            return this.rootPath;
+        }
+        return path.resolve(this.rootPath, relativePath);
+    }
+    async pruneEmptyDirectories(startDir) {
+        const normalizedStop = path.resolve(this.rootPath);
+        let current = path.resolve(startDir);
+        while (current !== normalizedStop && current.startsWith(`${normalizedStop}${path.sep}`)) {
+            let entries;
+            try {
+                entries = await fs.readdir(current);
+            }
+            catch {
+                return;
+            }
+            if (entries.length > 0) {
+                return;
+            }
+            try {
+                await fs.rmdir(current);
+            }
+            catch {
+                return;
+            }
+            current = path.dirname(current);
+        }
+    }
+}

package/dist/src/storage/types.js

@@ -0,0 +1,4 @@
+import path from "node:path";
+export function toPosix(value) {
+    return value.split(path.sep).join("/");
+}

package/dist/src/toolScope.js

@@ -0,0 +1,20 @@
+export const TOOL_SCOPE_LABELS = {
+    list_projects: "project_catalog",
+    recent_notes: "recent_listing",
+    search_notes: "search_listing",
+    read_note: "direct_read",
+    resolve_links: "link_audit",
+    list_backlinks: "backlink_audit",
+    context_bundle: "context_navigation",
+    memory_contract: "contract_reference",
+    lint_structure: "structure_audit",
+    write_note: "generic_write",
+    write_flow_note: "flow_write",
+    promote_to_stock: "flow_promotion",
+    append_note: "append_write",
+    delete_note: "delete_lifecycle",
+    rename_note: "rename_lifecycle",
+};
+export function responsibilityWarningCode(warnings) {
+    return warnings[0]?.code ?? "none";
+}

package/dist/src/types.js

@@ -0,0 +1 @@
+export {};