@llmkb/claude-code 0.1.0

package/lib/sync.ts

@@ -0,0 +1,437 @@
+/** File sync engine for ``llmkb sync``.
+
+Walks a directory, applies ``.gitignore`` + ``.llmkbignore`` filtering,
+computes SHA256 hashes, uploads new/changed files via TUS, and detects
+renames by content identity.
+
+Batch ingestion: Creates one ``IngestionJob`` upfront, uploads all files
+into it via ``/complete?postpone_ingestion=true&ingestion_job_id=``, then
+prints the job URL for the user to track progress in the UI.
+*/
+
+import { readdir, stat as fsStat, readFile } from "node:fs/promises";
+import { join, relative, resolve } from "node:path";
+import ignore from "ignore";
+import * as tus from "tus-js-client";
+import { existsSync } from "node:fs";
+import { getConfig } from "./config.js";
+import { getToken } from "./credentials.js";
+import { readSpaceConfig } from "./parser.js";
+import { computeSha256 } from "./sync-state.js";
+import { findProjectRoot } from "./parser.js";
+import type { SyncState, FileChangeSet } from "./types.js";
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const IGNORE_FILES = [".gitignore", ".llmkbignore"] as const;
+
+/** Default patterns always skipped (binary, hidden dirs, large artifacts, test artifacts).
+ *
+ * Python-side mirror at ``api/app/mcp/tools/entity_filter.py`` in the llmkb-ai repo.
+ * Keep both lists in sync. */
+const DEFAULT_SKIP_PATTERNS = [
+  "node_modules/**",
+  ".git/**",
+  ".llmkb/**",
+  ".claude/**",
+  ".cursor/**",
+  ".next/**",
+  "dist/**",
+  ".venv/**",
+  "__pycache__/**",
+  "*.pyc",
+  "*.exe",
+  "*.dll",
+  "*.so",
+  "*.dylib",
+  "*.bin",
+  // Test & ephemeral artifacts — exclude by default to avoid polluting
+  // the knowledge graph with test fixtures, snapshots, and build artifacts.
+  "**/__tests__/**",
+  "**/__snapshots__/**",
+  "**/test/**",
+  "**/tests/**",
+  "**/*.test.*",
+  "**/*.spec.*",
+  "**/fixtures/**",
+  "**/coverage/**",
+  "*.log",
+  "tmp/**",
+  "temp/**",
+];
+
+// ---------------------------------------------------------------------------
+// File Walking
+// ---------------------------------------------------------------------------
+
+export interface WalkOptions {
+  /** Project root directory (for ignore file lookup). */
+  projectDir: string;
+  /** Target directory to walk (defaults to projectDir). */
+  targetDir?: string;
+  /** Whether to respect .gitignore. */
+  respectGitignore?: boolean;
+  /** Whether to respect .llmkbignore. */
+  respectLlmkbignore?: boolean;
+}
+
+export interface WalkResult {
+  files: Array<[string, string]>; // [relativePath, absolutePath]
+  skipped: Array<{ path: string; reason: string }>;
+}
+
+/** Build an ignore filter from .gitignore, .llmkbignore, and default patterns.
+
+Reads ignore files from the project root first, then from the target
+directory — patterns from either source are applied.
+*/
+async function buildFilter(
+  projectDir: string,
+  targetDir: string,
+  respectGitignore: boolean,
+  respectLlmkbignore: boolean,
+): Promise<(path: string) => boolean> {
+  const ig = ignore().add(DEFAULT_SKIP_PATTERNS);
+
+  const searchDirs = [projectDir, targetDir];
+  const seen = new Set<string>();
+
+  for (const baseDir of searchDirs) {
+    for (const fileName of IGNORE_FILES) {
+      const shouldRespect =
+        fileName === ".gitignore" ? respectGitignore : respectLlmkbignore;
+      if (!shouldRespect) continue;
+
+      const filePath = join(baseDir, fileName);
+      if (seen.has(filePath)) continue;
+      seen.add(filePath);
+
+      if (existsSync(filePath)) {
+        // Skip directories that collide with ignore-file names
+        const st = await fsStat(filePath);
+        if (!st.isFile()) continue;
+        const content = await readFile(filePath, "utf-8");
+        ig.add(content);
+      }
+    }
+  }
+
+  return (testPath: string) => ig.ignores(testPath);
+}
+
+/** Recursively walk a directory, returning files filtered by ignore rules. */
+export async function walkFiles(
+  opts: WalkOptions,
+): Promise<WalkResult> {
+  const projectDir = resolve(opts.projectDir);
+  const targetDir = resolve(opts.targetDir ?? projectDir);
+  const respectGitignore = opts.respectGitignore ?? true;
+  const respectLlmkbignore = opts.respectLlmkbignore ?? true;
+
+  const isIgnored = await buildFilter(projectDir, targetDir, respectGitignore, respectLlmkbignore);
+  const files: Array<[string, string]> = [];
+  const skipped: Array<{ path: string; reason: string }> = [];
+
+  async function walk(dir: string): Promise<void> {
+    let entries;
+    try {
+      entries = await readdir(dir, { withFileTypes: true });
+    } catch {
+      skipped.push({ path: relative(targetDir, dir), reason: "unreadable" });
+      return;
+    }
+
+    for (const entry of entries) {
+      const fullPath = join(dir, entry.name);
+      const relPath = relative(targetDir, fullPath);
+
+      if (entry.name.startsWith(".") && !IGNORE_FILES.includes(entry.name as typeof IGNORE_FILES[number])) {
+        skipped.push({ path: relPath, reason: "hidden" });
+        continue;
+      }
+
+      if (isIgnored(relPath)) {
+        skipped.push({ path: relPath, reason: "ignored" });
+        continue;
+      }
+
+      if (entry.isDirectory()) {
+        // Recurse into directories
+        await walk(fullPath);
+      } else if (entry.isFile()) {
+        files.push([relPath, fullPath]);
+      } else if (entry.isSymbolicLink()) {
+        // Follow symlinks — stat to verify it's a file, not a dir symlink
+        try {
+          const st = await fsStat(fullPath);
+          if (st.isFile()) {
+            files.push([relPath, fullPath]);
+          } else {
+            skipped.push({ path: relPath, reason: "symlink-to-dir" });
+          }
+        } catch {
+          skipped.push({ path: relPath, reason: "broken-symlink" });
+        }
+      } else {
+        skipped.push({ path: relPath, reason: "not-a-file" });
+      }
+    }
+  }
+
+  await walk(targetDir);
+  return { files, skipped };
+}
+
+// ---------------------------------------------------------------------------
+// TUS Upload
+// ---------------------------------------------------------------------------
+
+export interface UploadOptions {
+  /** Target space name. */
+  space: string;
+  /** Relative path of the file within the space. */
+  relativePath: string;
+  /** Absolute path to the file on disk. */
+  absolutePath: string;
+  /** llmkb API endpoint. */
+  endpoint: string;
+  /** Auth token. */
+  token: string;
+  /** Upload progress callback. */
+  onProgress?: (bytesSent: number, bytesTotal: number) => void;
+}
+
+/** Upload a single file via TUS protocol (chunks only, no finalization). Returns the upload URL. */
+export async function tusUpload(
+  opts: UploadOptions,
+): Promise<string> {
+  // Safety check: verify it's a file, not a directory
+  const st = await fsStat(opts.absolutePath);
+  if (!st.isFile()) {
+    throw new Error(`Not a file: ${opts.relativePath}`);
+  }
+  const fileBuffer = await readFile(opts.absolutePath);
+  const fileName = opts.relativePath.split("/").pop() ?? opts.relativePath;
+
+  return new Promise<string>((resolve, reject) => {
+    const upload = new tus.Upload(
+      fileBuffer,
+      {
+        endpoint: `${opts.endpoint}/api/spaces/${opts.space}/uploads`,
+        metadata: {
+          filename: opts.relativePath,
+          filetype: "application/octet-stream",
+        },
+        headers: {
+          Authorization: `Bearer ${opts.token}`,
+        },
+        chunkSize: 5 * 1024 * 1024, // 5 MB chunks
+        retryDelays: [0, 1000, 3000, 5000],
+        removeFingerprintOnSuccess: true,
+        onError: (err: Error) => reject(err),
+        onProgress: (bytesSent: number, bytesTotal: number) => {
+          opts.onProgress?.(bytesSent, bytesTotal);
+        },
+        onSuccess: () => {
+          resolve(upload.url ?? "");
+        },
+      },
+    );
+    upload.start();
+  });
+}
+
+/** Finalize a TUS upload by calling /complete. Returns the upload URL on success. */
+export async function finalizeUpload(
+  uploadUrl: string,
+  token: string,
+  ingestionJobId?: string,
+): Promise<string> {
+  const params = new URLSearchParams({ postpone_ingestion: "true" });
+  if (ingestionJobId) {
+    params.set("ingestion_job_id", ingestionJobId);
+  }
+  const completeUrl = `${uploadUrl}/complete?${params.toString()}`;
+  const res = await fetch(completeUrl, {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${token}`,
+      "Tus-Resumable": "1.0.0",
+    },
+  });
+  if (!res.ok) {
+    const body = await res.text().catch(() => "");
+    const shortId = uploadUrl.split("/").pop() ?? "unknown";
+    throw new Error(`Upload completion failed for ${shortId} (${res.status}): ${body}`);
+  }
+  return uploadUrl;
+}
+
+// ---------------------------------------------------------------------------
+// Sync Engine
+// ---------------------------------------------------------------------------
+
+export interface SyncOptions {
+  /** Path to sync (directory or file). */
+  path: string;
+  /** Comma-separated list of skip reasons to ignore (for --dry-run filtering). */
+  skipReasons?: string[];
+  /** Callback for each file upload attempt. */
+  onFile?: (relPath: string, status: "uploading" | "skipped" | "renamed" | "unchanged" | "failed", detail?: string) => void;
+}
+
+export interface SyncResult {
+  uploaded: number;
+  skipped: number;
+  unchanged: number;
+  renamed: number;
+  errors: Array<{ path: string; error: string }>;
+  /** Ingestion job ID(s) after batch finalization. */
+  ingestionJobIds?: string[];
+  /** URL of the first ingestion job (for the sync command to display). */
+  ingestionJobUrl?: string;
+  /** Machine-readable outcome: "synced" | "no_files" | "up_to_date". */
+  status?: "synced" | "no_files" | "up_to_date";
+  /** Human-readable explanation when status is not "synced". */
+  reason?: string;
+}
+
+/** Run the full sync: walk -> compare -> upload -> update state. */
+export async function runSync(
+  space: string,
+  syncState: SyncState | null,
+  opts: SyncOptions,
+): Promise<SyncResult> {
+  const projectRoot = findProjectRoot();
+  if (!projectRoot) throw new Error("No .llmkb/ directory found. Run `llmkb init` first.");
+
+  const config = getConfig();
+  const endpoint = config.endpoint;
+  const token = await getToken();
+
+  const result: SyncResult = { uploaded: 0, skipped: 0, unchanged: 0, renamed: 0, errors: [] };
+
+  // Walk files
+  const walkResult = await walkFiles({
+    projectDir: projectRoot,
+    targetDir: resolve(opts.path),
+  });
+
+  for (const s of walkResult.skipped) {
+    result.skipped++;
+    opts.onFile?.(s.path, "skipped", s.reason);
+  }
+
+  if (walkResult.files.length === 0) {
+    result.status = "no_files";
+    const filteredCount = walkResult.skipped.length;
+    if (filteredCount > 0) {
+      result.reason = `No files to sync — all ${filteredCount} file(s) were filtered out by .gitignore / .llmkbignore rules.`;
+    } else {
+      result.reason = "No files found in sync target. The directory appears to be empty.";
+    }
+    return result;
+  }
+
+  // Classify files
+  const changes = await import("./sync-state.js").then((m) =>
+    m.getChangedFiles(walkResult.files, syncState),
+  );
+
+  // Process unchanged
+  for (const f of changes.unchangedFiles) {
+    result.unchanged++;
+    opts.onFile?.(f, "unchanged");
+  }
+
+  // Process renamed
+  for (const r of changes.renamed) {
+    result.renamed++;
+    opts.onFile?.(r.newPath, "renamed", `${r.oldPath} → ${r.newPath}`);
+  }
+
+  // Upload new + changed — collect upload URLs for batch finalization
+  const toUpload = [...changes.newFiles, ...changes.changedFiles];
+  if (toUpload.length === 0) {
+    result.status = "up_to_date";
+    result.reason = `No changes detected — all files are up to date (${walkResult.files.length} files, unchanged: ${result.unchanged}, renamed: ${result.renamed}, skipped: ${result.skipped}).`;
+    return result;
+  }
+  const uploadUrls: string[] = [];
+  const ingestionJobId: string | undefined = undefined;
+  // No pre-flight job creation — start_ingestion_batch on the backend
+  // handles job creation and document association. Passing undefined as
+  // ingestionJobId means /complete sets ingestion_job_id=NULL, and the
+  // batch endpoint picks up all pending docs regardless.
+
+  for (const relPath of toUpload) {
+    const absPath = join(resolve(opts.path), relPath);
+
+    if (!token) {
+      result.errors.push({ path: relPath, error: "No token found" });
+      opts.onFile?.(relPath, "failed", "no token");
+      continue;
+    }
+
+    try {
+      const absEntry = walkResult.files.find(([, a]) => a === absPath);
+      const absPathResolved = absEntry?.[1] ?? absPath;
+      const url = await tusUpload({
+        space,
+        relativePath: relPath,
+        absolutePath: absPathResolved,
+        endpoint,
+        token,
+      });
+      uploadUrls.push(url);
+      result.uploaded++;
+      opts.onFile?.(relPath, "uploading");
+    } catch (err) {
+      result.errors.push({ path: relPath, error: (err as Error).message });
+      opts.onFile?.(relPath, "failed", (err as Error).message);
+    }
+  }
+
+  // Finalize all uploads into the pre-created job
+  if (uploadUrls.length > 0) {
+    for (const url of uploadUrls) {
+      try {
+        await finalizeUpload(url, token!, ingestionJobId);
+      } catch (err) {
+        const shortId = url.split("/").pop() ?? "unknown";
+        result.errors.push({ path: shortId, error: (err as Error).message });
+        opts.onFile?.(shortId, "failed", (err as Error).message);
+      }
+    }
+    // Trigger the ingestion pipeline for all finalized documents
+    if (token) {
+      try {
+        const ingestRes = await fetch(
+          `${endpoint}/api/spaces/${space}/uploads/start-ingest`,
+          {
+            method: "POST",
+            headers: { Authorization: `Bearer ${token}` },
+          },
+        );
+        if (ingestRes.ok) {
+          const ingestBody = (await ingestRes.json()) as {
+            data?: { attributes?: { job_id?: string; url?: string } };
+          };
+          // Use the real job URL from the batch endpoint (more accurate
+          // than the pre-flight placeholder URL).
+          const realUrl = ingestBody.data?.attributes?.url;
+          if (realUrl) {
+            result.ingestionJobUrl = realUrl;
+          }
+        }
+      } catch {
+        // Non-fatal — documents are stored, ingestion can be triggered manually
+      }
+    }
+  }
+
+  result.status = "synced";
+  return result;
+}

package/lib/types.ts

@@ -0,0 +1,153 @@
+/** Shared TypeScript interfaces for the llmkb plugin config model.
+
+These types define the schema for the per-project configuration files
+(``.llmkb/spaces.yml`` and ``.llmkb/config.yml``) used by all CLI commands,
+the MCP server config, and the sync engine.
+
+Conventions:
+- All field names follow the YAML key naming (lower_snake_case).
+- Optional fields are marked with ``?``.
+- Tokens MUST NOT appear in any config file — they live in the OS keychain or env vars.
+*/
+
+// ---------------------------------------------------------------------------
+// Space Definition (new PSM-4 format)
+// ---------------------------------------------------------------------------
+
+/** A project space entry — the primary space for file sync. */
+export interface ProjectSpaceDef {
+  /** Space UUID. */
+  id: string;
+  /** Optional human-readable name. */
+  name?: string;
+  /** Optional slug for MCP tool names. */
+  slug?: string;
+  /** Optional directory paths to sync from this project. */
+  dirs?: string[];
+}
+
+/** A referenced space entry (additional spaces beyond project_space). */
+export interface SpaceDef {
+  /** Space UUID. */
+  id: string;
+  /** Optional human-readable name. */
+  name?: string;
+  /** Optional slug for MCP tool names. */
+  slug?: string;
+}
+
+// ---------------------------------------------------------------------------
+// Full Space Config (``.llmkb/spaces.yml``)
+// ---------------------------------------------------------------------------
+
+export interface SpaceConfig {
+  /** The project's primary space (file sync target). */
+  project_space?: ProjectSpaceDef[];
+  /** Additional spaces this project has access to. */
+  spaces?: SpaceDef[];
+}
+
+// ---------------------------------------------------------------------------
+// Plugin Behavior Config (``.llmkb/config.yml``)
+// ---------------------------------------------------------------------------
+
+export interface WatchSettings {
+  enabled?: boolean;
+  debounce_ms?: number;
+  gitignore?: boolean;
+  llmkbignore?: boolean;
+  default_verbosity?: "silent" | "verbose" | "debug";
+}
+
+export interface SyncSettings {
+  concurrency?: number;
+  retry_attempts?: number;
+  retry_delay_ms?: number;
+}
+
+export interface HookSettings {
+  timeout_ms?: number;
+  skip?: boolean;
+}
+
+export interface PluginConfig {
+  /** Backend API base URL (e.g. https://api.llmkb.ai). */
+  llmkb_base_url?: string;
+  watch?: WatchSettings;
+  sync?: SyncSettings;
+  hook?: HookSettings;
+  debug?: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// Version Stamp
+// ---------------------------------------------------------------------------
+
+/** The version file at ``.llmkb/.llmkb-version`` stores just the semver string. */
+
+// ---------------------------------------------------------------------------
+// Exported utilities
+// ---------------------------------------------------------------------------
+
+export const PACKAGE_VERSION = "0.1.0";
+
+/** Returns the default plugin behavior config used when ``config.yml`` is missing or all fields are commented. */
+export function getDefaultPluginConfig(): PluginConfig {
+  return {
+    llmkb_base_url: "https://api.llmkb.ai",
+    watch: {
+      enabled: true,
+      debounce_ms: 300,
+      gitignore: true,
+      llmkbignore: true,
+      default_verbosity: "silent",
+    },
+    sync: {
+      concurrency: 5,
+      retry_attempts: 3,
+      retry_delay_ms: 1000,
+    },
+    hook: {
+      timeout_ms: 5000,
+      skip: false,
+    },
+    debug: false,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Token Types
+// ---------------------------------------------------------------------------
+
+/** The only token type used by the plugin. */
+export type TokenType = "access_token";
+
+// ---------------------------------------------------------------------------
+// Sync State Types
+// ---------------------------------------------------------------------------
+
+export type SyncFileStatus = "synced" | "pending" | "failed";
+
+export interface SyncFileEntry {
+  sha256: string;
+  lastUploaded: string | null;
+  status: SyncFileStatus;
+}
+
+export interface SyncState {
+  space: string;
+  lastSyncAt: string;
+  files: Record<string, SyncFileEntry>;
+}
+
+export interface RenameEntry {
+  oldPath: string;
+  newPath: string;
+}
+
+export interface FileChangeSet {
+  newFiles: string[];
+  changedFiles: string[];
+  unchangedFiles: string[];
+  renamed: RenameEntry[];
+}

package/lib/watch-lock.ts

@@ -0,0 +1,78 @@
+/** Advisory file lock for chokidar watch mode.
+
+Prevents duplicate watchers in the same project. Uses a simple PID-file
+pattern: writes the current PID to ``.llmkb/.watch.lock`` and checks
+liveness of the owning process before granting the lock.
+*/
+
+import { writeFile, readFile, unlink } from "node:fs/promises";
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { spawnSync } from "node:child_process";
+
+const LOCK_FILE = ".llmkb/.watch.lock";
+
+/**
+ * Acquire an advisory watch lock.
+ *
+ * Returns ``true`` if the lock was acquired (or if no project root was found).
+ * Returns ``false`` if another watcher holds the lock and its PID is still alive.
+ */
+export async function acquireWatchLock(projectRoot: string): Promise<boolean> {
+  if (!projectRoot) return true;
+  const lockPath = join(projectRoot, LOCK_FILE);
+
+  if (existsSync(lockPath)) {
+    try {
+      const content = await readFile(lockPath, "utf-8");
+      const pid = parseInt(content.trim(), 10);
+
+      if (Number.isFinite(pid) && pid > 0) {
+        try {
+          // Check if process is alive (ESRCH = dead, EPERM = alive but cross-user)
+          process.kill(pid, 0);
+          // Process is alive — lock is held
+          return false;
+        } catch (e) {
+          // ESRCH means process is gone — we can take the lock
+          if ((e as NodeJS.ErrnoException).code !== "ESRCH") {
+            return false;
+          }
+        }
+      }
+    } catch {
+      // Unreadable or corrupt lock — allow override
+    }
+  }
+
+  await writeFile(lockPath, String(process.pid), "utf-8");
+  return true;
+}
+
+/**
+ * Release the advisory watch lock.
+ */
+export async function releaseWatchLock(projectRoot: string): Promise<void> {
+  if (!projectRoot) return;
+  const lockPath = join(projectRoot, LOCK_FILE);
+
+  try {
+    // Only unlink if we still own it
+    if (existsSync(lockPath)) {
+      const content = await readFile(lockPath, "utf-8");
+      if (content.trim() === String(process.pid)) {
+        await unlink(lockPath);
+      }
+    }
+  } catch {
+    // Best-effort cleanup
+  }
+}
+
+/**
+ * Check whether a watch lock currently exists.
+ */
+export function hasWatchLock(projectRoot: string): boolean {
+  if (!projectRoot) return false;
+  return existsSync(join(projectRoot, LOCK_FILE));
+}