@skyphusion/search-mcp 0.1.2

package/scripts/sync-ingest.mjs

@@ -0,0 +1,141 @@
+// AI Search ingest planning: which git paths need a .txt suffix on the R2 key.
+//
+// Cloudflare AI Search keys off the object name extension. Source we already remap
+// (.ts, .jsx, ...) plus extensionless text (Dockerfile) and text files whose
+// extension is not in the supported list (.service, .example, ...) upload as
+// <path>.txt so the indexer treats them as plain text.
+
+import { extname, basename } from "node:path";
+
+/** Extensions AI Search ingests natively (plain + rich). See CF data-source docs. */
+export const AI_SEARCH_NATIVE_EXT = new Set([
+  ".txt", ".rst",
+  ".log",
+  ".ini", ".conf", ".properties", ".toml",
+  ".markdown", ".md", ".mdx", ".mdoc",
+  ".tex", ".latex",
+  ".sh", ".bat", ".ps1",
+  ".sgml",
+  ".json",
+  ".sql",
+  ".yaml", ".yml",
+  ".css",
+  ".js",
+  ".php",
+  ".py",
+  ".rb",
+  ".java",
+  ".c", ".cpp", ".cxx", ".h", ".hpp",
+  ".go",
+  ".rs",
+  ".swift",
+  ".dart",
+  ".el",
+  ".pdf",
+  ".html", ".htm",
+  ".xml",
+  ".xlsx", ".xlsm", ".xlsb", ".xls", ".et", ".docx",
+  ".ods", ".odt",
+  ".csv",
+  ".numbers",
+]);
+
+/** Basenames with no extname() that AI Search lists as native config types. */
+export const AI_SEARCH_NATIVE_BASENAMES = new Set([
+  ".gitignore",
+  ".editorconfig",
+  ".dockerignore",
+  ".env",
+]);
+
+/** Always append .txt regardless of content sniff (TS/JSX + common infra/config suffixes). */
+export const REMAP_TO_TXT_EXPLICIT = new Set([
+  ".ts", ".tsx", ".mts", ".cts", ".jsx", ".astro", ".vue", ".svelte",
+  ".mjs", ".cjs",
+  ".jsonc",
+  ".example", ".sample", ".template", ".tmpl", ".local",
+  ".webmanifest", ".manifest",
+  ".service", ".socket", ".timer", ".mount", ".path", ".target", ".slice", ".automount",
+  ".v4", ".v6",
+]);
+
+export const DOC_EXT = new Set([".md", ".markdown", ".mdx", ".mdoc", ".rst", ".txt", ".pdf"]);
+
+export const CONTENT_TYPE = {
+  ".txt": "text/plain", ".rst": "text/plain",
+  ".md": "text/markdown", ".markdown": "text/markdown", ".mdx": "text/markdown",
+  ".json": "application/json", ".yaml": "application/x-yaml", ".yml": "application/x-yaml",
+  ".py": "text/x-python", ".go": "text/x-go", ".rs": "text/rust",
+  ".js": "application/javascript", ".mjs": "application/javascript", ".cjs": "application/javascript",
+  ".sh": "application/x-sh", ".bash": "application/x-sh",
+  ".sql": "application/sql", ".toml": "text/plain", ".ini": "text/plain", ".conf": "text/plain",
+  ".html": "text/html", ".htm": "text/html", ".xml": "application/xml",
+  ".css": "text/css", ".csv": "text/csv", ".pdf": "application/pdf",
+};
+
+export function fileExt(relPath) {
+  return extname(basename(relPath)).toLowerCase();
+}
+
+/** True when the first bytes look like text (no NUL, mostly valid UTF-8). */
+export function isLikelyText(buf) {
+  if (!buf || buf.length === 0) return true;
+  if (buf.includes(0)) return false;
+  const s = buf.toString("utf8");
+  const bad = (s.match(/\uFFFD/g) || []).length;
+  return bad <= Math.max(1, Math.floor(buf.length * 0.02));
+}
+
+export function isNativeIngestPath(relPath) {
+  const base = basename(relPath);
+  const ext = fileExt(relPath);
+  return AI_SEARCH_NATIVE_EXT.has(ext) || AI_SEARCH_NATIVE_BASENAMES.has(base);
+}
+
+/**
+ * Should the R2 object key get a trailing .txt?
+ * Native paths: false. Explicit remap extensions: true. Otherwise: true when sample is text.
+ */
+export function shouldRemapToTxt(relPath, sample) {
+  if (isNativeIngestPath(relPath)) return false;
+  const ext = fileExt(relPath);
+  if (REMAP_TO_TXT_EXPLICIT.has(ext)) return true;
+  return isLikelyText(sample);
+}
+
+/** True when the file should be uploaded (native as-is, or remapped text). */
+export function isIngestible(relPath, sample) {
+  if (isNativeIngestPath(relPath)) return true;
+  return shouldRemapToTxt(relPath, sample);
+}
+
+export function ingestObjectKey(repo, relPath, remapped) {
+  return `${repo}/${relPath}${remapped ? ".txt" : ""}`;
+}
+
+export function ingestContentType(relPath, remapped) {
+  const ext = remapped ? ".txt" : fileExt(relPath);
+  return CONTENT_TYPE[ext] || "text/plain";
+}
+
+export function ingestKind(relPath) {
+  const ext = fileExt(relPath);
+  return DOC_EXT.has(ext) || basename(relPath).startsWith("README") ? "doc" : "code";
+}
+
+/**
+ * Per-repo corpus exclusion (targets.json `excludePaths`). An entry is a repo-relative
+ * path: with a trailing "/" it excludes the whole subtree; without, it excludes that
+ * exact file or the subtree rooted at that name.
+ */
+export function isExcludedPath(relPath, prefixes) {
+  for (const p of prefixes || []) {
+    if (!p) continue;
+    if (p.endsWith("/")) {
+      if (relPath.startsWith(p)) return true;
+    } else if (relPath === p || relPath.startsWith(p + "/")) {
+      return true;
+    }
+  }
+  return false;
+}

package/scripts/sync-runner.mjs

@@ -0,0 +1,213 @@
+#!/usr/bin/env node
+// Corpus sync runner: isolated clone root -> scripts/sync.mjs -> optional reindex.
+//
+// Usage:
+//   node scripts/sync-runner.mjs                 # all targets in targets.json
+//   node scripts/sync-runner.mjs corpus          # one target
+//   node scripts/sync-runner.mjs --dry-run
+//   node scripts/sync-runner.mjs --no-reindex
+//   node scripts/sync-runner.mjs --no-github-verify
+//
+// Env:
+//   CORPUS_ROOT, CORPUS_GIT_ORG (required for clone)
+//   GITHUB_TOKEN | GH_TOKEN
+//   R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, R2_ACCOUNT_ID | CLOUDFLARE_ACCOUNT_ID
+//   CLOUDFLARE_API_TOKEN + CLOUDFLARE_ACCOUNT_ID (reindex)
+
+import { execFileSync } from "node:child_process";
+import { existsSync, mkdirSync, readFileSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import {
+  resolveTargetsPath,
+  targetsHelp,
+} from "./config-paths.mjs";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const REPO = join(HERE, "..");
+
+/** Parse argv into a normalized options object (pure). */
+export function parseArgs(argv, allTargetNames = []) {
+  const flags = new Set(argv.filter((a) => a.startsWith("--")));
+  const positional = argv.filter((a) => !a.startsWith("--"));
+  const requested = positional[0];
+  return {
+    targets: requested ? [requested] : [...allTargetNames],
+    dryRun: flags.has("--dry-run"),
+    reindex: !flags.has("--no-reindex") && !flags.has("--dry-run"),
+    passThrough: [...flags].filter((f) => f === "--dry-run" || f === "--no-github-verify"),
+  };
+}
+
+export function reposForTargets(cfg, targets) {
+  const seen = new Set();
+  const repos = [];
+  for (const t of targets) {
+    const target = cfg?.targets?.[t];
+    if (!target) throw new Error(`unknown target ${t}`);
+    for (const r of target.repos || []) {
+      if (!seen.has(r)) {
+        seen.add(r);
+        repos.push(r);
+      }
+    }
+  }
+  return repos;
+}
+
+export function instancesForTargets(cfg, targets) {
+  return targets.map((t) => cfg.targets[t].instance);
+}
+
+export function gitAuthArgs(token) {
+  if (!token) return [];
+  const basic = Buffer.from(`x-access-token:${token}`).toString("base64");
+  return ["-c", `http.https://github.com/.extraheader=AUTHORIZATION: basic ${basic}`];
+}
+
+export function scrubSecrets(s) {
+  return String(s)
+    .replace(/AUTHORIZATION:\s*basic\s+\S+/gi, "AUTHORIZATION: basic [REDACTED]")
+    .replace(/Bearer\s+\S+/gi, "Bearer [REDACTED]");
+}
+
+export function describeExit(err) {
+  const status = err && (err.status ?? err.code);
+  return scrubSecrets(`exit ${status ?? "unknown"}`);
+}
+
+function repoUrl(org, repo) {
+  return `https://github.com/${org}/${repo}.git`;
+}
+
+function git(args, opts = {}) {
+  return execFileSync("git", args, { stdio: ["ignore", "pipe", "inherit"], ...opts })
+    .toString()
+    .trim();
+}
+
+function syncRepoTree(org, repo, root, authArgs) {
+  const dir = join(root, repo);
+  const url = repoUrl(org, repo);
+  if (!existsSync(join(dir, ".git"))) {
+    git([...authArgs, "clone", "--quiet", url, dir]);
+  } else {
+    git([...authArgs, "-C", dir, "remote", "set-url", "origin", url]);
+    git([...authArgs, "-C", dir, "fetch", "--prune", "--quiet", "origin"]);
+  }
+  git([...authArgs, "-C", dir, "remote", "set-head", "origin", "--auto"]);
+  let head;
+  try {
+    head = git(["-C", dir, "symbolic-ref", "--short", "refs/remotes/origin/HEAD"]);
+  } catch {
+    head = "origin/main";
+  }
+  git(["-C", dir, "reset", "--hard", "--quiet", head]);
+  git(["-C", dir, "clean", "-fdq"]);
+}
+
+function reindexInstance(instance) {
+  execFileSync("npx", ["wrangler", "ai-search", "jobs", "create", instance, "--json"], {
+    stdio: "inherit",
+    cwd: REPO,
+  });
+}
+
+export function run(plan, deps) {
+  const { repos, targets, instances, reindex } = plan;
+  const { cloneTree, runSync, runReindex, log = console.log, error = console.error } = deps;
+
+  log(`Refreshing ${repos.length} repo tree(s) ...`);
+  const failedClones = [];
+  for (const repo of repos) {
+    try {
+      cloneTree(repo);
+      log(`  ok  ${repo}`);
+    } catch (err) {
+      failedClones.push(repo);
+      error(`  !!  ${repo}: git failed (${describeExit(err)})`);
+    }
+  }
+
+  if (failedClones.length) {
+    error(
+      `BLOCKER: ${failedClones.length} clone/fetch failure(s) [${failedClones.join(", ")}]; ` +
+        "aborting before sync so no repo corpus is pruned or synced stale.",
+    );
+    return { exitCode: 1, synced: false };
+  }
+
+  for (const target of targets) {
+    log(`\n=== sync ${target} ===`);
+    runSync(target);
+  }
+
+  const failedReindex = [];
+  if (reindex) {
+    for (const instance of instances) {
+      log(`\n=== reindex ${instance} ===`);
+      try {
+        runReindex(instance);
+      } catch (err) {
+        failedReindex.push(instance);
+        error(`  !! reindex failed for ${instance} (${describeExit(err)})`);
+      }
+    }
+  } else {
+    log("\nReindex skipped; AI Search will pick up R2 changes on its next schedule.");
+  }
+
+  if (failedReindex.length) {
+    error(`BLOCKER: ${failedReindex.length} reindex failure(s) [${failedReindex.join(", ")}].`);
+    return { exitCode: 1, synced: true };
+  }
+  log("\nCorpus sync complete.");
+  return { exitCode: 0, synced: true };
+}
+
+function loadConfig() {
+  const path = resolveTargetsPath(HERE);
+  if (!path) {
+    console.error(targetsHelp(HERE));
+    process.exit(2);
+  }
+  return JSON.parse(readFileSync(path, "utf8"));
+}
+
+function main() {
+  const cfg = loadConfig();
+  const allTargets = Object.keys(cfg.targets || {});
+  const opts = parseArgs(process.argv.slice(2), allTargets);
+  const repos = reposForTargets(cfg, opts.targets);
+  const instances = instancesForTargets(cfg, opts.targets);
+  const root = process.env.CORPUS_ROOT || join(REPO, ".corpus");
+  const org = process.env.CORPUS_GIT_ORG || "";
+  if (!org) {
+    console.error("Set CORPUS_GIT_ORG to the GitHub org that owns the repos in targets.json.");
+    process.exit(2);
+  }
+  const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN || "";
+  const authArgs = gitAuthArgs(token);
+
+  mkdirSync(root, { recursive: true });
+  console.log(`Corpus root: ${root}`);
+  console.log(`Org: ${org}`);
+
+  const result = run(
+    { repos, targets: opts.targets, instances, reindex: opts.reindex },
+    {
+      cloneTree: (repo) => syncRepoTree(org, repo, root, authArgs),
+      runSync: (target) =>
+        execFileSync("node", [join(HERE, "sync.mjs"), target, ...opts.passThrough], {
+          stdio: "inherit",
+          cwd: REPO,
+          env: { ...process.env, SYNC_REPO_ROOT: root },
+        }),
+      runReindex: reindexInstance,
+    },
+  );
+  process.exit(result.exitCode);
+}
+
+const invokedDirectly = process.argv[1] && process.argv[1] === fileURLToPath(import.meta.url);
+if (invokedDirectly) main();

package/scripts/sync.mjs

@@ -0,0 +1,288 @@
+#!/usr/bin/env node
+// Git-to-R2 sync for Cloudflare AI Search.
+//
+// Walks git-tracked files for each repo in a target, filters binaries / secrets /
+// build noise, remaps source AI Search cannot natively index to .txt, attaches
+// metadata, uploads to the target R2 bucket, and prunes stale objects (mirror).
+//
+// Usage:
+//   node scripts/sync.mjs corpus
+//   node scripts/sync.mjs corpus --dry-run
+//   node scripts/sync.mjs public --no-github-verify
+//
+// Env:
+//   R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY   (required unless --dry-run)
+//   R2_ACCOUNT_ID or CLOUDFLARE_ACCOUNT_ID   (to build the S3 endpoint)
+//   R2_S3_ENDPOINT                            (optional override)
+//   SEARCH_MCP_TARGETS                       (optional path to targets.json)
+//   SYNC_REPO_ROOT                            (optional clone root)
+//   GITHUB_TOKEN or GH_TOKEN                  (optional; live visibility check for public target)
+//   CORPUS_GIT_ORG                            (required for GitHub visibility check)
+//   SYNC_SKIP_GITHUB_VERIFY=1                 (optional)
+
+import { execFileSync } from "node:child_process";
+import { readFileSync, existsSync, statSync, openSync, readSync, closeSync } from "node:fs";
+import { join, dirname, basename } from "node:path";
+import { fileURLToPath } from "node:url";
+import {
+  assertPublicCorpusBoundary,
+  assertPublicGithubVisibility,
+} from "./corpus-boundary.mjs";
+import {
+  isIngestible,
+  shouldRemapToTxt,
+  isNativeIngestPath,
+  ingestObjectKey,
+  ingestContentType,
+  ingestKind,
+  fileExt,
+  isExcludedPath,
+} from "./sync-ingest.mjs";
+import {
+  resolveTargetsPath,
+  defaultRepoRoot,
+  targetsHelp,
+} from "./config-paths.mjs";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const TARGETS_PATH = resolveTargetsPath(HERE);
+const REPO_ROOT = defaultRepoRoot(HERE, TARGETS_PATH);
+const MAX_BYTES = 4 * 1024 * 1024;
+
+const args = process.argv.slice(2);
+const DRY = args.includes("--dry-run");
+const SKIP_GITHUB = args.includes("--no-github-verify") || process.env.SYNC_SKIP_GITHUB_VERIFY === "1";
+const targetName = args.find((a) => !a.startsWith("--"));
+
+const SKIP_EXT = new Set([
+  ".woff", ".woff2", ".ttf", ".otf", ".eot", ".ico", ".icns",
+  ".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tiff", ".psd",
+  ".mp4", ".mov", ".webm", ".mkv", ".avi", ".mp3", ".wav", ".flac", ".ogg",
+  ".zip", ".gz", ".tgz", ".tar", ".bz2", ".xz", ".7z", ".rar",
+  ".bin", ".exe", ".dll", ".so", ".dylib", ".wasm", ".o", ".a",
+  ".pt", ".pth", ".safetensors", ".onnx", ".gguf", ".ckpt", ".npy", ".npz", ".parquet",
+  ".age", ".gpg", ".enc", ".pem", ".crt", ".p12", ".pfx", ".jks",
+  ".map", ".lock",
+]);
+
+const SKIP_BASENAME = new Set([
+  "package-lock.json", "pnpm-lock.yaml", "yarn.lock",
+  "poetry.lock", "Cargo.lock", "uv.lock", "bun.lockb",
+]);
+
+const SKIP_DIR_SEGMENT = new Set([
+  "node_modules", "dist", "build", ".wrangler", "coverage", ".git",
+  "vendor", ".venv", "venv", "target", "__pycache__", ".next", ".turbo", ".cache",
+]);
+
+function isEnvSecret(base) {
+  if (!/^\.env($|\.)/.test(base) && !/\.env$/.test(base)) return false;
+  return !/(example|sample|template)/i.test(base);
+}
+
+function shouldSkip(relPath) {
+  const segs = relPath.split("/");
+  if (segs.some((s) => SKIP_DIR_SEGMENT.has(s))) return true;
+  const base = basename(relPath);
+  if (SKIP_BASENAME.has(base)) return true;
+  if (isEnvSecret(base)) return true;
+  const ext = fileExt(relPath);
+  if (SKIP_EXT.has(ext)) return true;
+  if (base.endsWith(".min.js")) return true;
+  return false;
+}
+
+function trackedFiles(repoDir) {
+  const out = execFileSync("git", ["ls-files", "-z"], {
+    cwd: repoDir,
+    maxBuffer: 256 * 1024 * 1024,
+  });
+  return out.toString("utf8").split("\0").filter(Boolean);
+}
+
+const TEXT_SAMPLE_BYTES = 8192;
+
+function readTextSample(abs, size) {
+  const len = Math.min(TEXT_SAMPLE_BYTES, size);
+  if (len === 0) return Buffer.alloc(0);
+  const fd = openSync(abs, "r");
+  try {
+    const buf = Buffer.alloc(len);
+    readSync(fd, buf, 0, len, 0);
+    return buf;
+  } finally {
+    closeSync(fd);
+  }
+}
+
+function loadConfig() {
+  if (!TARGETS_PATH) {
+    console.error(targetsHelp(HERE));
+    process.exit(2);
+  }
+  return JSON.parse(readFileSync(TARGETS_PATH, "utf8"));
+}
+
+function planRepo(repo, excludePrefixes = []) {
+  const repoDir = join(REPO_ROOT, repo);
+  if (!existsSync(repoDir)) {
+    console.warn(`  ! skip ${repo}: not cloned at ${repoDir}`);
+    return [];
+  }
+  const items = [];
+  for (const rel of trackedFiles(repoDir)) {
+    if (shouldSkip(rel)) continue;
+    if (isExcludedPath(rel, excludePrefixes)) continue;
+    const abs = join(repoDir, rel);
+    let size;
+    try {
+      size = statSync(abs).size;
+    } catch {
+      continue;
+    }
+    if (size > MAX_BYTES) {
+      console.warn(`  ! skip ${repo}/${rel}: ${(size / 1048576).toFixed(1)} MB over 4 MB`);
+      continue;
+    }
+    const sample = readTextSample(abs, size);
+    if (!isIngestible(rel, sample)) continue;
+    const remapped = !isNativeIngestPath(rel) && shouldRemapToTxt(rel, sample);
+    const origExt = fileExt(rel);
+    const key = ingestObjectKey(repo, rel, remapped);
+    items.push({
+      abs,
+      key,
+      contentType: ingestContentType(rel, remapped),
+      metadata: {
+        repo,
+        path: rel.slice(0, 500),
+        lang: (origExt.replace(".", "") || "text").slice(0, 60),
+        kind: ingestKind(rel),
+      },
+    });
+  }
+  return items;
+}
+
+async function main() {
+  if (!targetName) {
+    console.error("usage: node scripts/sync.mjs <target> [--dry-run] [--no-github-verify]");
+    process.exit(2);
+  }
+  const cfg = loadConfig();
+  const target = cfg.targets[targetName];
+  if (!target) {
+    console.error(`unknown target '${targetName}'. known: ${Object.keys(cfg.targets).join(", ")}`);
+    process.exit(2);
+  }
+
+  assertPublicCorpusBoundary(cfg, targetName);
+  if (targetName === "public" && !SKIP_GITHUB) {
+    const gh = await assertPublicGithubVisibility(cfg);
+    if (gh.skipped) {
+      console.warn(
+        "  ! GITHUB_TOKEN / GH_TOKEN unset: skipping live GitHub visibility check " +
+          "(overlap check passed). Set a token or pass --no-github-verify to silence.",
+      );
+    } else {
+      console.log(`  GitHub visibility ok for ${gh.checked.length} public-target repos.`);
+    }
+  } else if (targetName === "public" && SKIP_GITHUB) {
+    console.warn("  ! --no-github-verify: live GitHub visibility check skipped.");
+  }
+
+  console.log(`Target '${targetName}' -> bucket ${target.bucket} (instance ${target.instance})`);
+  console.log(`Repo root: ${REPO_ROOT}${DRY ? "  [DRY RUN]" : ""}`);
+
+  const plan = [];
+  for (const repo of target.repos) {
+    const items = planRepo(repo, cfg.excludePaths?.[repo]);
+    if (items.length) console.log(`  + ${repo}: ${items.length} files`);
+    plan.push(...items);
+  }
+  console.log(`Planned ${plan.length} objects for ${target.bucket}.`);
+
+  if (DRY) {
+    for (const it of plan.slice(0, 25)) console.log(`    ${it.key}`);
+    if (plan.length > 25) console.log(`    ... and ${plan.length - 25} more`);
+    console.log("Dry run: no uploads, no prune.");
+    return;
+  }
+
+  const { S3Client, PutObjectCommand, ListObjectsV2Command, DeleteObjectsCommand } =
+    await import("@aws-sdk/client-s3");
+
+  const accountId = process.env.R2_ACCOUNT_ID || process.env.CLOUDFLARE_ACCOUNT_ID;
+  const endpoint =
+    process.env.R2_S3_ENDPOINT ||
+    (accountId ? `https://${accountId}.r2.cloudflarestorage.com` : undefined);
+  if (!endpoint) {
+    console.error("Set R2_S3_ENDPOINT or R2_ACCOUNT_ID / CLOUDFLARE_ACCOUNT_ID.");
+    process.exit(2);
+  }
+  if (!process.env.R2_ACCESS_KEY_ID || !process.env.R2_SECRET_ACCESS_KEY) {
+    console.error("Set R2_ACCESS_KEY_ID and R2_SECRET_ACCESS_KEY.");
+    process.exit(2);
+  }
+
+  const s3 = new S3Client({
+    region: "auto",
+    endpoint,
+    credentials: {
+      accessKeyId: process.env.R2_ACCESS_KEY_ID,
+      secretAccessKey: process.env.R2_SECRET_ACCESS_KEY,
+    },
+  });
+  const Bucket = target.bucket;
+
+  const seen = new Set();
+  let up = 0;
+  let idx = 0;
+  const concurrency = Math.max(1, Number(process.env.UPLOAD_CONCURRENCY || 12));
+  async function uploadWorker() {
+    while (idx < plan.length) {
+      const it = plan[idx++];
+      await s3.send(
+        new PutObjectCommand({
+          Bucket,
+          Key: it.key,
+          Body: readFileSync(it.abs),
+          ContentType: it.contentType,
+          Metadata: it.metadata,
+        }),
+      );
+      seen.add(it.key);
+      if (++up % 200 === 0) console.log(`  uploaded ${up}/${plan.length}`);
+    }
+  }
+  await Promise.all(
+    Array.from({ length: Math.min(concurrency, plan.length) }, uploadWorker),
+  );
+  console.log(`Uploaded ${up} objects.`);
+
+  const stale = [];
+  let ContinuationToken;
+  do {
+    const res = await s3.send(
+      new ListObjectsV2Command({ Bucket, ContinuationToken }),
+    );
+    for (const obj of res.Contents || []) {
+      if (obj.Key && !seen.has(obj.Key)) stale.push({ Key: obj.Key });
+    }
+    ContinuationToken = res.IsTruncated ? res.NextContinuationToken : undefined;
+  } while (ContinuationToken);
+
+  for (let i = 0; i < stale.length; i += 1000) {
+    const batch = stale.slice(i, i + 1000);
+    await s3.send(
+      new DeleteObjectsCommand({ Bucket, Delete: { Objects: batch } }),
+    );
+  }
+  console.log(`Pruned ${stale.length} stale objects.`);
+  console.log("Done. AI Search will re-index on its next sync.");
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

package/scripts/targets.json.example

@@ -0,0 +1,14 @@
+{
+  "_comment": "Copy to targets.json (./targets.json for npm CLI, or scripts/targets.json in a git clone). repos are directory names under SYNC_REPO_ROOT (default: cwd for npm, parent of this repo for dev clone). Only git-tracked files are uploaded.",
+  "excludePaths": {
+    "example-repo": ["vendor/private-notes/"]
+  },
+  "restrictedRepos": [],
+  "targets": {
+    "corpus": {
+      "instance": "my-ai-search-instance",
+      "bucket": "my-search-corpus",
+      "repos": ["example-repo"]
+    }
+  }
+}