@fractary/codex-cli 0.3.1 → 0.4.1

package/README.md

@@ -42,7 +42,7 @@ Options:
 **Example:**
 ```bash
 fractary-codex init
-# Creates .fractary/codex.yaml and .codex-cache/
+# Creates .fractary/codex/config.yaml and .fractary/codex/cache/
 ```
 
 ### `fetch` - Fetch Documents
@@ -153,6 +153,41 @@ Options:
   --parallel <n>       Number of parallel sync operations (default: 3)
 ```
 
+**Routing-Aware Sync (v4.1+):**
+
+When using `--from-codex` direction, the sync command uses **routing-aware file discovery** to find all files across the entire codex that should sync to your project based on `codex_sync_include` frontmatter patterns.
+
+How it works:
+1. Clones the entire codex repository to a temporary directory (`/tmp/fractary-codex-clone/`)
+2. Scans ALL markdown files in the codex recursively
+3. Evaluates `codex_sync_include` patterns in each file's frontmatter
+4. Returns only files that match your project name or pattern
+
+**Example frontmatter in source files:**
+```yaml
+---
+codex_sync_include: ['*']                  # Syncs to ALL projects
+codex_sync_include: ['lake-*', 'api-*']    # Syncs to lake-* and api-* projects
+codex_sync_exclude: ['*-test']             # Except *-test projects
+---
+```
+
+**Trade-offs:**
+- **Efficient for discovery**: Finds all relevant files across hundreds of projects
+- **Inefficient for execution**: Clones entire codex repository (uses shallow clone for speed)
+- **Best practice**: Use `codex://` URI references + cache purging for most workflows; use sync occasionally when needed
+
+**Recommended workflow:**
+```bash
+# Primary: Reference files via codex:// URIs (no sync needed)
+fractary-codex fetch codex://org/project/path/file.md
+
+# When you need latest versions: Purge cache
+fractary-codex cache clear --pattern "codex://org/project/*"
+
+# Then MCP will re-fetch fresh content automatically
+```
+
 ### `types` - Type Registry Management
 
 Manage custom artifact types for classification and caching.
@@ -234,11 +269,11 @@ Options:
 
 ## Configuration
 
-Codex uses `.fractary/codex.yaml` for configuration:
+Codex uses `.fractary/codex/config.yaml` for configuration:
 
 ```yaml
 organization: myorg
-cacheDir: .codex-cache
+cacheDir: .fractary/codex/cache
 
 storage:
   - type: github
@@ -328,16 +363,6 @@ All commands use lazy-loading to avoid SDK initialization overhead for simple op
 
 MIT
 
-## Documentation
-
-- [Command Reference](../docs/guides/command-reference.md) - Complete command reference for all interfaces
-- [API Reference](../docs/guides/api-reference.md) - Complete API documentation
-- [CLI Integration Guide](../docs/guides/cli-integration.md) - How to integrate into CLI applications
-- [Configuration Guide](../docs/guides/configuration.md) - Configuration reference
-- [Naming Conventions](../docs/guides/naming-conventions.md) - Naming standards across all interfaces
-- [MCP Migration Guide](../docs/guides/mcp-migration-guide.md) - Migrating to new MCP tool names
-- [Troubleshooting](../docs/guides/troubleshooting.md) - Common issues and solutions
-
 ## Links
 
 - [GitHub Repository](https://github.com/fractary/codex)

package/dist/cli.cjs

@@ -5,6 +5,8 @@ var fs = require('fs/promises');
 var path3 = require('path');
 var yaml = require('js-yaml');
 var codex = require('@fractary/codex');
+var os = require('os');
+var child_process = require('child_process');
 var commander = require('commander');
 var chalk8 = require('chalk');
 var crypto = require('crypto');
@@ -32,6 +34,7 @@ function _interopNamespace(e) {
 var fs__namespace = /*#__PURE__*/_interopNamespace(fs);
 var path3__namespace = /*#__PURE__*/_interopNamespace(path3);
 var yaml__namespace = /*#__PURE__*/_interopNamespace(yaml);
+var os__namespace = /*#__PURE__*/_interopNamespace(os);
 var chalk8__default = /*#__PURE__*/_interopDefault(chalk8);
 var crypto__namespace = /*#__PURE__*/_interopNamespace(crypto);
 
@@ -90,7 +93,7 @@ async function migrateConfig(legacyConfigPath, options) {
       organization: legacy.organization || legacy.organizationSlug || "default"
     };
     if (legacy.cache) {
-      yamlConfig.cacheDir = legacy.cache.directory || ".codex-cache";
+      yamlConfig.cacheDir = legacy.cache.directory || ".fractary/codex/cache";
     }
     if (legacy.storage?.providers) {
       yamlConfig.storage = [];
@@ -199,7 +202,7 @@ async function writeYamlConfig(config, outputPath) {
 function getDefaultYamlConfig(organization) {
   return {
     organization,
-    cacheDir: ".codex-cache",
+    cacheDir: ".fractary/codex/cache",
     storage: [
       {
         type: "local",
@@ -619,6 +622,178 @@ var init_codex_client = __esm({
   }
 });
 
+// src/utils/codex-repository.ts
+var codex_repository_exports = {};
+__export(codex_repository_exports, {
+  ensureCodexCloned: () => ensureCodexCloned,
+  getCodexRepoUrl: () => getCodexRepoUrl,
+  getTempCodexPath: () => getTempCodexPath,
+  isValidGitRepo: () => isValidGitRepo
+});
+function spawnAsync(command, args, options) {
+  return new Promise((resolve, reject) => {
+    const child = child_process.spawn(command, args, {
+      ...options,
+      env: process.env
+    });
+    let stdout = "";
+    let stderr = "";
+    child.stdout.on("data", (data) => {
+      stdout += data.toString();
+    });
+    child.stderr.on("data", (data) => {
+      stderr += data.toString();
+    });
+    child.on("error", (error) => {
+      reject(error);
+    });
+    child.on("close", (code) => {
+      if (code !== 0) {
+        const error = new Error(stderr || `Command exited with code ${code}`);
+        error.code = code;
+        reject(error);
+      } else {
+        resolve(stdout);
+      }
+    });
+  });
+}
+function sanitizePathComponent(component) {
+  if (!component || typeof component !== "string") {
+    throw new Error("Path component must be a non-empty string");
+  }
+  const sanitized = component.replace(/\.\./g, "").replace(/[/\\]/g, "").trim();
+  if (!sanitized) {
+    throw new Error(`Invalid path component: ${component}`);
+  }
+  return sanitized;
+}
+function validateGitHubName(name, type) {
+  if (!name || typeof name !== "string") {
+    throw new Error(`GitHub ${type} name must be a non-empty string`);
+  }
+  if (!/^[a-zA-Z0-9._-]+$/.test(name)) {
+    throw new Error(`Invalid GitHub ${type} name: ${name}. Must contain only alphanumeric characters, hyphens, underscores, and dots.`);
+  }
+  if (name.startsWith(".") || name.startsWith("-")) {
+    throw new Error(`GitHub ${type} name cannot start with a dot or hyphen: ${name}`);
+  }
+}
+function getTempCodexPath(config) {
+  const codexRepo = config.codex_repository || "codex";
+  const sanitizedOrg = sanitizePathComponent(config.organization);
+  const sanitizedRepo = sanitizePathComponent(codexRepo);
+  return path3__namespace.join(
+    os__namespace.tmpdir(),
+    "fractary-codex-clone",
+    `${sanitizedOrg}-${sanitizedRepo}-${process.pid}`
+  );
+}
+async function isValidGitRepo(repoPath) {
+  try {
+    const gitDir = path3__namespace.join(repoPath, ".git");
+    const stats = await fs__namespace.stat(gitDir);
+    return stats.isDirectory();
+  } catch {
+    return false;
+  }
+}
+function getCodexRepoUrl(config) {
+  const codexRepo = config.codex_repository || "codex";
+  validateGitHubName(config.organization, "organization");
+  validateGitHubName(codexRepo, "repository");
+  return `https://github.com/${config.organization}/${codexRepo}.git`;
+}
+async function execGit(repoPath, args) {
+  try {
+    const stdout = await spawnAsync("git", args, {
+      cwd: repoPath
+    });
+    return stdout;
+  } catch (error) {
+    if (error.code === "ENOENT") {
+      throw new Error(`Git command not found. Ensure git is installed and in PATH.`);
+    }
+    if (error.code === "EACCES") {
+      throw new Error(`Permission denied accessing repository at ${repoPath}`);
+    }
+    if (error.code === 128) {
+      throw new Error(`Git authentication failed. Check your credentials and repository access.`);
+    }
+    throw error;
+  }
+}
+async function gitClone(url, targetPath, options) {
+  const parentDir = path3__namespace.dirname(targetPath);
+  await fs__namespace.mkdir(parentDir, { recursive: true });
+  const args = ["clone"];
+  if (options?.depth) {
+    if (!Number.isInteger(options.depth) || options.depth <= 0) {
+      throw new Error(`Invalid depth parameter: ${options.depth}. Must be a positive integer.`);
+    }
+    args.push("--depth", String(options.depth));
+  }
+  if (options?.branch) {
+    if (!/^[\w\-./]+$/.test(options.branch)) {
+      throw new Error(`Invalid branch name: ${options.branch}`);
+    }
+    args.push("--branch", options.branch);
+  }
+  args.push("--single-branch");
+  args.push(url, targetPath);
+  await spawnAsync("git", args, { cwd: parentDir });
+}
+async function gitFetch(repoPath, branch) {
+  {
+    if (!/^[\w\-./]+$/.test(branch)) {
+      throw new Error(`Invalid branch name: ${branch}`);
+    }
+    await execGit(repoPath, ["fetch", "origin", `${branch}:${branch}`]);
+  }
+}
+async function gitCheckout(repoPath, branch) {
+  if (!/^[\w\-./]+$/.test(branch)) {
+    throw new Error(`Invalid branch name: ${branch}`);
+  }
+  await execGit(repoPath, ["checkout", branch]);
+}
+async function gitPull(repoPath) {
+  await execGit(repoPath, ["pull"]);
+}
+async function ensureCodexCloned(config, options) {
+  const tempPath = getTempCodexPath(config);
+  const branch = options?.branch || "main";
+  if (await isValidGitRepo(tempPath) && !options?.force) {
+    try {
+      await gitFetch(tempPath, branch);
+      await gitCheckout(tempPath, branch);
+      await gitPull(tempPath);
+      return tempPath;
+    } catch (error) {
+      console.warn(`Failed to update existing clone: ${error.message}`);
+      console.warn(`Removing and cloning fresh...`);
+      await fs__namespace.rm(tempPath, { recursive: true, force: true });
+    }
+  }
+  const repoUrl = getCodexRepoUrl(config);
+  try {
+    await fs__namespace.rm(tempPath, { recursive: true, force: true });
+  } catch (error) {
+    console.warn(`Could not remove existing directory ${tempPath}: ${error.message}`);
+  }
+  await gitClone(repoUrl, tempPath, {
+    branch,
+    depth: 1
+    // Shallow clone for efficiency
+  });
+  return tempPath;
+}
+var init_codex_repository = __esm({
+  "src/utils/codex-repository.ts"() {
+    init_cjs_shims();
+  }
+});
+
 // src/cli.ts
 init_cjs_shims();
 
@@ -767,24 +942,18 @@ function initCommand() {
         console.log(chalk8__default.default.dim(`Organization: ${chalk8__default.default.cyan(org)}
 `));
       }
-      const configDir = path3__namespace.join(process.cwd(), ".fractary");
-      const configPath = path3__namespace.join(configDir, "codex.yaml");
+      const configDir = path3__namespace.join(process.cwd(), ".fractary", "codex");
+      const configPath = path3__namespace.join(configDir, "config.yaml");
       const configExists = await fileExists(configPath);
-      const legacyConfigPath = path3__namespace.join(process.cwd(), ".fractary", "plugins", "codex", "config.json");
-      const legacyExists = await fileExists(legacyConfigPath);
       if (configExists && !options.force) {
-        console.log(chalk8__default.default.yellow("\u26A0 Configuration already exists at .fractary/codex.yaml"));
+        console.log(chalk8__default.default.yellow("\u26A0 Configuration already exists at .fractary/codex/config.yaml"));
         console.log(chalk8__default.default.dim("Use --force to overwrite"));
         process.exit(1);
       }
-      if (legacyExists && !configExists) {
-        console.log(chalk8__default.default.yellow("\u26A0 Legacy configuration detected at .fractary/plugins/codex/config.json"));
-        console.log(chalk8__default.default.dim('Run "fractary codex migrate" to upgrade to YAML format\n'));
-      }
       console.log("Creating directory structure...");
       const dirs = [
-        ".fractary",
-        ".codex-cache"
+        ".fractary/codex",
+        ".fractary/codex/cache"
       ];
       for (const dir of dirs) {
         await fs__namespace.mkdir(path3__namespace.join(process.cwd(), dir), { recursive: true });
@@ -796,12 +965,12 @@ function initCommand() {
         config.mcp.enabled = true;
       }
       await writeYamlConfig(config, configPath);
-      console.log(chalk8__default.default.green("\u2713"), chalk8__default.default.dim(".fractary/codex.yaml"));
-      console.log(chalk8__default.default.green("\n\u2713 Codex v3.0 initialized successfully!\n"));
+      console.log(chalk8__default.default.green("\u2713"), chalk8__default.default.dim(".fractary/codex/config.yaml"));
+      console.log(chalk8__default.default.green("\n\u2713 Codex v4.0 initialized successfully!\n"));
       console.log(chalk8__default.default.bold("Configuration:"));
       console.log(chalk8__default.default.dim(`  Organization: ${org}`));
-      console.log(chalk8__default.default.dim(`  Cache: .codex-cache/`));
-      console.log(chalk8__default.default.dim(`  Config: .fractary/codex.yaml`));
+      console.log(chalk8__default.default.dim(`  Cache: .fractary/codex/cache/`));
+      console.log(chalk8__default.default.dim(`  Config: .fractary/codex/config.yaml`));
       if (options.mcp) {
         console.log(chalk8__default.default.dim(`  MCP Server: Enabled (port 3000)`));
       }
@@ -811,13 +980,9 @@ function initCommand() {
       console.log(chalk8__default.default.dim("  - HTTP endpoint"));
       console.log(chalk8__default.default.bold("\nNext steps:"));
       console.log(chalk8__default.default.dim('  1. Set your GitHub token: export GITHUB_TOKEN="your_token"'));
-      console.log(chalk8__default.default.dim("  2. Edit .fractary/codex.yaml to configure storage providers"));
+      console.log(chalk8__default.default.dim("  2. Edit .fractary/codex/config.yaml to configure storage providers"));
       console.log(chalk8__default.default.dim("  3. Fetch a document: fractary codex fetch codex://org/project/path"));
       console.log(chalk8__default.default.dim("  4. Check cache: fractary codex cache list"));
-      if (legacyExists) {
-        console.log(chalk8__default.default.yellow("\n\u26A0 Legacy config detected:"));
-        console.log(chalk8__default.default.dim('  Run "fractary codex migrate" to convert your existing config'));
-      }
     } catch (error) {
       console.error(chalk8__default.default.red("Error:"), error.message);
       process.exit(1);
@@ -1450,7 +1615,7 @@ function syncCommand() {
       let projectName = name;
       if (!projectName) {
         const detected = detectCurrentProject();
-        projectName = detected.project || null;
+        projectName = detected.project || void 0;
       }
       if (!projectName) {
         console.error(chalk8__default.default.red("Error:"), "Could not determine project name.");
@@ -1473,7 +1638,7 @@ function syncCommand() {
         config: config.sync,
         manifestPath: path3__namespace.join(process.cwd(), ".fractary", ".codex-sync-manifest.json")
       });
-      const defaultPatterns = config.sync?.include || [
+      const defaultPatterns = [
         "docs/**/*.md",
         "specs/**/*.md",
         ".fractary/standards/**",
@@ -1508,13 +1673,71 @@ function syncCommand() {
         include: includePatterns,
         exclude: excludePatterns
       };
-      const plan = await syncManager.createPlan(
-        config.organization,
-        projectName,
-        sourceDir,
-        targetFiles,
-        syncOptions
-      );
+      let plan;
+      let routingScan;
+      if (direction === "from-codex") {
+        let codexRepoPath;
+        try {
+          const { ensureCodexCloned: ensureCodexCloned2 } = await Promise.resolve().then(() => (init_codex_repository(), codex_repository_exports));
+          if (!options.json) {
+            console.log(chalk8__default.default.blue("\u2139 Cloning/updating codex repository..."));
+          }
+          codexRepoPath = await ensureCodexCloned2(config, {
+            branch: targetBranch
+          });
+          if (!options.json) {
+            console.log(chalk8__default.default.dim(`  Codex cloned to: ${codexRepoPath}`));
+            console.log(chalk8__default.default.dim("  Scanning for files routing to this project...\n"));
+          } else {
+            console.log(JSON.stringify({
+              info: "Routing-aware sync: cloned codex repository and scanning for files targeting this project",
+              codexPath: codexRepoPath
+            }, null, 2));
+          }
+        } catch (error) {
+          console.error(chalk8__default.default.red("Error:"), "Failed to clone codex repository");
+          console.error(chalk8__default.default.dim(`  ${error.message}`));
+          console.log(chalk8__default.default.yellow("\nTroubleshooting:"));
+          if (error.message.includes("Git command not found")) {
+            console.log(chalk8__default.default.dim("  Git is not installed or not in PATH."));
+            console.log(chalk8__default.default.dim("  Install git: https://git-scm.com/downloads"));
+          } else if (error.message.includes("authentication failed") || error.message.includes("Authentication failed")) {
+            console.log(chalk8__default.default.dim("  GitHub authentication is required for private repositories."));
+            console.log(chalk8__default.default.dim("  1. Check auth status: gh auth status"));
+            console.log(chalk8__default.default.dim("  2. Login if needed: gh auth login"));
+            console.log(chalk8__default.default.dim("  3. Or set GITHUB_TOKEN environment variable"));
+          } else if (error.message.includes("Permission denied")) {
+            console.log(chalk8__default.default.dim("  Permission denied accessing repository files."));
+            console.log(chalk8__default.default.dim("  1. Check file/directory permissions"));
+            console.log(chalk8__default.default.dim("  2. Ensure you have access to the repository"));
+          } else if (error.message.includes("not found") || error.message.includes("does not exist")) {
+            console.log(chalk8__default.default.dim(`  Repository not found: ${config.organization}/${config.codex_repository || "codex"}`));
+            console.log(chalk8__default.default.dim("  1. Verify the repository exists on GitHub"));
+            console.log(chalk8__default.default.dim("  2. Check organization and repository names in config"));
+          } else {
+            console.log(chalk8__default.default.dim("  1. Ensure git is installed: git --version"));
+            console.log(chalk8__default.default.dim("  2. Check GitHub auth: gh auth status"));
+            console.log(chalk8__default.default.dim(`  3. Verify repo exists: ${config.organization}/${config.codex_repository || "codex"}`));
+          }
+          process.exit(1);
+        }
+        const planWithRouting = await syncManager.createRoutingAwarePlan(
+          config.organization,
+          projectName,
+          codexRepoPath,
+          syncOptions
+        );
+        plan = planWithRouting;
+        routingScan = planWithRouting.routingScan;
+      } else {
+        plan = await syncManager.createPlan(
+          config.organization,
+          projectName,
+          sourceDir,
+          targetFiles,
+          syncOptions
+        );
+      }
       if (plan.totalFiles === 0) {
         if (options.json) {
           console.log(JSON.stringify({
@@ -1577,6 +1800,20 @@ function syncCommand() {
       if (plan.estimatedTime) {
         console.log(`  Est. time:    ${chalk8__default.default.dim(formatDuration(plan.estimatedTime))}`);
       }
+      if (routingScan) {
+        console.log("");
+        console.log(chalk8__default.default.bold("Routing Statistics\n"));
+        console.log(`  Scanned:      ${chalk8__default.default.cyan(routingScan.stats.totalScanned.toString())} files`);
+        console.log(`  Matched:      ${chalk8__default.default.cyan(routingScan.stats.totalMatched.toString())} files`);
+        console.log(`  Source projects: ${chalk8__default.default.cyan(routingScan.stats.sourceProjects.length.toString())}`);
+        if (routingScan.stats.sourceProjects.length > 0) {
+          console.log(chalk8__default.default.dim(`    ${routingScan.stats.sourceProjects.slice(0, 5).join(", ")}`));
+          if (routingScan.stats.sourceProjects.length > 5) {
+            console.log(chalk8__default.default.dim(`    ... and ${routingScan.stats.sourceProjects.length - 5} more`));
+          }
+        }
+        console.log(`  Scan time:    ${chalk8__default.default.dim(formatDuration(routingScan.stats.durationMs))}`);
+      }
       console.log("");
       if (plan.conflicts.length > 0) {
         console.log(chalk8__default.default.yellow(`\u26A0 ${plan.conflicts.length} conflicts detected:`));