@sanjay5114/cdx 1.0.0

package/lib/scanner.js

@@ -0,0 +1,351 @@
+"use strict";
+
+/**
+ * CDX File Scanner & Metadata Pipeline
+ *
+ * Stages: SCAN → FILTER → PARSE_METADATA → DETECT_STACK
+ *         → PRIORITIZE → SCORE → SELECT_TOP_N → CHUNK → COMPRESS → TOKEN_CHECK
+ */
+
+const fs   = require("fs").promises;
+const path = require("path");
+
+// ─── Constants ────────────────────────────────────────────────────────────────
+const CODE_EXTS = new Set([
+  ".js",".ts",".jsx",".tsx",".mjs",".cjs",
+  ".py",".go",".rs",".java",".c",".cpp",".h",".hpp",
+  ".rb",".php",".swift",".kt",".cs",".dart",
+  ".json",".yaml",".yml",".toml",
+  ".md",".mdx",".txt",
+  ".html",".htm",".css",".scss",".sass",".less",".vue",".svelte",".astro",".svg",
+  ".sh",".bash",".zsh",
+  ".env.example",
+  "dockerfile","makefile","procfile",
+]);
+
+const IGNORE_DIRS = new Set([
+  "node_modules",".git","build","dist","out","assets",
+  ".next",".nuxt","coverage","__pycache__",".venv","venv",
+  ".cache",".turbo",".parcel-cache","storybook-static",
+  "public","static","vendor",".gradle","target",
+]);
+
+const IGNORE_FILES = new Set([
+  ".DS_Store","Thumbs.db","package-lock.json","yarn.lock","pnpm-lock.yaml",
+  ".npmrc",".nvmrc",".node-version",".editorconfig",
+]);
+
+const SENSITIVE_RX = [
+  /\.env$/i,/secret/i,/credential/i,/password/i,
+  /private[_-]?key/i,/\.pem$/i,/\.key$/i,/\.pfx$/i,
+  /\.p12$/i,/id_rsa/i,/id_ed25519/i,/\.cert$/i,
+];
+
+const MAX_FILE_BYTES  = 500_000;
+const TOP_N           = 25;
+const CHUNK_LINES     = 300;
+const TOKEN_BUDGET    = 12_000;
+const TOKENS_PER_META = 60;
+
+// ─── Sensitivity ──────────────────────────────────────────────────────────────
+function isSensitive(filePath) {
+  const base = path.basename(filePath).toLowerCase();
+  return SENSITIVE_RX.some(rx => rx.test(base));
+}
+
+// ─── Local file scan ──────────────────────────────────────────────────────────
+async function scanLocal(dir, includeHidden = false, files = []) {
+  let entries;
+  try { entries = await fs.readdir(dir, { withFileTypes: true }); }
+  catch { return files; }
+
+  for (const ent of entries) {
+    if (!includeHidden && ent.name.startsWith(".")) continue;
+    if (IGNORE_DIRS.has(ent.name.toLowerCase()) || IGNORE_FILES.has(ent.name)) continue;
+    const fullPath = path.join(dir, ent.name);
+    if (ent.isDirectory()) {
+      await scanLocal(fullPath, includeHidden, files);
+    } else {
+      const ext    = path.extname(ent.name).toLowerCase();
+      const base   = ent.name.toLowerCase();
+      if (CODE_EXTS.has(ext) || CODE_EXTS.has(base)) {
+        try {
+          const { size } = await fs.stat(fullPath);
+          if (size <= MAX_FILE_BYTES) files.push({ path: fullPath, size });
+        } catch {}
+      }
+    }
+  }
+  return files;
+}
+
+// ─── Parse metadata from local file ──────────────────────────────────────────
+async function parseLocal(filePath) {
+  const isSens   = isSensitive(filePath);
+  const content  = isSens ? "" : await fs.readFile(filePath, "utf8").catch(() => "");
+  return buildMeta(path.relative(process.cwd(), filePath), path.extname(filePath), content, isSens);
+}
+
+// ─── Parse metadata from raw content string ──────────────────────────────────
+function buildMeta(file, ext, content = "", sensitive = false) {
+  const lines      = content ? content.split("\n") : [];
+  const fnMatches  = content.match(/function\s+\w+|\b\w+\s*[:=]\s*(?:async\s*)?\(|(?:def|fn|func)\s+\w+|\w+\s*\(.*\)\s*[{:]/g) || [];
+  const importCnt  = (content.match(/\b(?:require|import)\b/g)  || []).length;
+  const exportCnt  = (content.match(/\b(?:module\.exports|export\s+(?:default|const|function|class))\b/g) || []).length;
+  const complexity = (content.match(/\b(?:if|else|for|while|switch|catch|&&|\|\||\?)\b/g) || []).length;
+  const classCnt   = (content.match(/\bclass\s+\w+/g) || []).length;
+  const asyncCnt   = (content.match(/\basync\b/g) || []).length;
+  const testFile   = /\.(test|spec)\.[jt]sx?$/.test(file) || /__(tests?|mocks?)__/.test(file);
+
+  return {
+    file, ext: ext.toLowerCase(),
+    size     : lines.length,
+    functions: Math.min(fnMatches.length, 999),
+    imports  : importCnt,
+    exports  : exportCnt,
+    complexity,
+    classes  : classCnt,
+    asyncOps : asyncCnt,
+    testFile, sensitive,
+    chunked  : false,
+  };
+}
+
+// ─── Stack Detection ──────────────────────────────────────────────────────────
+async function detectStack(metas, opts = {}) {
+  const files    = metas.map(m => m.file.toLowerCase());
+  const hasFile  = (rx) => files.some(f => rx.test(f));
+  const extCount = (ext) => metas.filter(m => m.ext === ext).length;
+
+  const checks = [
+    { stack: "next",    score: (hasFile(/next\.config/) ? 10 : 0) + (hasFile(/\/pages\/|\/app\//) ? 5 : 0) },
+    { stack: "react",   score: (extCount(".jsx") + extCount(".tsx")) * 2 },
+    { stack: "vue",     score: extCount(".vue") * 3 },
+    { stack: "svelte",  score: extCount(".svelte") * 3 },
+    { stack: "angular", score: hasFile(/\.component\.ts$/) ? 10 : 0 },
+    { stack: "flutter", score: extCount(".dart") * 3 + (hasFile(/pubspec\.yaml/) ? 5 : 0) },
+    { stack: "python",  score: extCount(".py") * 2 + (hasFile(/requirements\.txt|setup\.py|pyproject\.toml/) ? 4 : 0) },
+    { stack: "go",      score: extCount(".go") * 2 + (hasFile(/go\.mod|go\.sum/) ? 4 : 0) },
+    { stack: "rust",    score: extCount(".rs") * 2 + (hasFile(/cargo\.toml/) ? 4 : 0) },
+    { stack: "java",    score: extCount(".java") * 2 + (hasFile(/pom\.xml|build\.gradle/) ? 4 : 0) },
+    { stack: "node",    score: hasFile(/package\.json/) ? 4 : 0 },
+  ];
+
+  checks.sort((a, b) => b.score - a.score);
+  const top    = checks[0];
+  const stack  = top.score > 2 ? top.stack : "generic";
+  const reasons = [];
+
+  if (hasFile(/next\.config/))                  reasons.push("next.config found");
+  if (hasFile(/package\.json/))                 reasons.push("Node.js project");
+  if (extCount(".py") > 0)                      reasons.push(`${extCount(".py")} Python files`);
+  if (extCount(".go") > 0)                      reasons.push(`${extCount(".go")} Go files`);
+  if (extCount(".dart") > 0)                    reasons.push(`${extCount(".dart")} Dart files`);
+  if (hasFile(/pubspec\.yaml/))                 reasons.push("Flutter pubspec.yaml");
+  if (extCount(".jsx") + extCount(".tsx") > 0)  reasons.push(`${extCount(".jsx") + extCount(".tsx")} React components`);
+
+  return { stack, reasons: reasons.length ? reasons : ["generic codebase"] };
+}
+
+// ─── Stack-aware priority boosting ───────────────────────────────────────────
+const STACK_PATTERNS = {
+  next   : [/^pages\/|^app\/|next\.config|middleware\./],
+  react  : [/^src\/|components\/|hooks\/|context\//],
+  vue    : [/\.vue$|store\//],
+  flutter: [/^lib\/|\.dart$/],
+  python : [/\.py$/],
+  go     : [/\.go$|main\.go|cmd\//],
+  node   : [/^src\/|server\.|app\.|index\.js|routes?\//],
+  generic: [],
+};
+
+function prioritize(metas, stack) {
+  const patterns = STACK_PATTERNS[stack] || [];
+  if (!patterns.length) return metas;
+  const priority = metas.filter(m => patterns.some(rx => rx.test(m.file.toLowerCase())));
+  const rest     = metas.filter(m => !priority.includes(m));
+  return [...priority, ...rest];
+}
+
+// ─── Scoring ──────────────────────────────────────────────────────────────────
+function score(metas) {
+  return metas.map(m => ({
+    ...m,
+    score:
+      m.functions  * 3.0 +
+      m.classes    * 4.0 +
+      m.exports    * 2.5 +
+      m.imports    * 1.5 +
+      m.complexity * 1.5 +
+      m.asyncOps   * 1.0 +
+      Math.min(m.size / 60, 8) +
+      (m.testFile   ? -6  : 0) +
+      (m.sensitive  ? -20 : 0) +
+      // Bonus for key filenames
+      (/index\.|main\.|app\.|server\./.test(m.file) ? 5 : 0) +
+      (/config|setting|env/.test(m.file.toLowerCase()) ? 3 : 0),
+  }));
+}
+
+function selectTopN(scored, n = TOP_N) {
+  return [...scored].sort((a, b) => b.score - a.score).slice(0, n);
+}
+
+function chunkLarge(metas) {
+  return metas.map(m => m.size > CHUNK_LINES ? { ...m, chunked: true } : m);
+}
+
+function compress(metas) {
+  return metas.map(({ file, ext, size, functions, imports, exports, complexity, classes, asyncOps, testFile, sensitive, chunked }) =>
+    ({ file, ext, size, functions, imports, exports, complexity, classes, asyncOps, testFile: !!testFile, sensitive: !!sensitive, chunked: !!chunked }));
+}
+
+function tokenCheck(metas) {
+  return metas.slice(0, Math.floor(TOKEN_BUDGET / TOKENS_PER_META));
+}
+
+// ─── Full local pipeline ──────────────────────────────────────────────────────
+async function runLocalPipeline(cwd, opts = {}) {
+  const files = await scanLocal(cwd, opts.includeHidden || false);
+  if (!files.length) throw new Error("No source files found in this directory.");
+
+  const metas = [];
+  for (const f of files) {
+    try { metas.push(await parseLocal(f.path)); }
+    catch {}
+  }
+
+  const stackInfo    = await detectStack(metas);
+  const prioritized  = prioritize(metas, stackInfo.stack);
+  const scored       = score(prioritized);
+  const top          = selectTopN(scored, TOP_N);
+  const chunked      = chunkLarge(top);
+  const compressed   = compress(chunked);
+  const selected     = tokenCheck(compressed);
+
+  return { raw: metas, selected, stackInfo };
+}
+
+// ─── GitHub remote pipeline ───────────────────────────────────────────────────
+const fetch = require("./fetch");
+
+async function githubRequest(token, method, endpoint, body) {
+  const res = await fetch(`https://api.github.com${endpoint}`, {
+    method,
+    headers: {
+      Authorization          : `Bearer ${token}`,
+      Accept                 : "application/vnd.github+json",
+      "Content-Type"         : "application/json",
+      "User-Agent"           : "CDX-CLI/3.0",
+      "X-GitHub-Api-Version" : "2022-11-28",
+    },
+    body  : body ? JSON.stringify(body) : undefined,
+    signal: AbortSignal.timeout(20_000),
+  });
+  if (!res.ok && res.status !== 404) {
+    const txt = await res.text().catch(() => "");
+    throw new Error(`GitHub API ${res.status}: ${txt.slice(0, 300)}`);
+  }
+  return res.status === 204 ? null : res.json().catch(() => null);
+}
+
+async function runRemotePipeline(owner, repo, token) {
+  const repoData = await githubRequest(token, "GET", `/repos/${owner}/${repo}`);
+  if (!repoData) throw new Error("Could not fetch remote repository info.");
+  const branch = repoData.default_branch || "main";
+
+  const treeData = await githubRequest(token, "GET", `/repos/${owner}/${repo}/git/trees/${branch}?recursive=1`);
+  if (!treeData?.tree) throw new Error("Could not fetch repository tree.");
+
+  const blobs = treeData.tree.filter(f => f.type === "blob");
+  const metas = [];
+
+  for (const f of blobs) {
+    const parts = f.path.split("/");
+    if (parts.some(p => IGNORE_DIRS.has(p.toLowerCase()))) continue;
+    if (IGNORE_FILES.has(path.basename(f.path))) continue;
+
+    const ext  = path.extname(f.path).toLowerCase();
+    const base = path.basename(f.path).toLowerCase();
+    if (!CODE_EXTS.has(ext) && !CODE_EXTS.has(base)) continue;
+
+    const isSens  = isSensitive(f.path);
+    let content   = "";
+
+    if (!isSens && (f.size || 0) <= MAX_FILE_BYTES) {
+      const blob = await githubRequest(token, "GET", `/repos/${owner}/${repo}/git/blobs/${f.sha}`).catch(() => null);
+      if (blob?.content) {
+        content = Buffer.from(blob.content, blob.encoding || "base64").toString("utf8");
+      }
+    }
+    metas.push(buildMeta(f.path, ext, content, isSens));
+  }
+
+  if (!metas.length) throw new Error("No source files found in remote repository.");
+
+  const stackInfo   = await detectStack(metas);
+  const prioritized = prioritize(metas, stackInfo.stack);
+  const scored      = score(prioritized);
+  const top         = selectTopN(scored, TOP_N);
+  const chunked     = chunkLarge(top);
+  const compressed  = compress(chunked);
+  const selected    = tokenCheck(compressed);
+
+  return { raw: metas, selected, stackInfo };
+}
+
+// ─── GitHub push ──────────────────────────────────────────────────────────────
+async function upsertFile(token, owner, repo, filePath, content, msg) {
+  const endpoint = `/repos/${owner}/${repo}/contents/${filePath}`;
+  const existing = await githubRequest(token, "GET", endpoint).catch(() => null);
+  const sha      = existing?.sha;
+  await githubRequest(token, "PUT", endpoint, {
+    message: msg,
+    content: Buffer.from(content, "utf8").toString("base64"),
+    ...(sha ? { sha } : {}),
+  });
+}
+
+async function pushToGitHub(token, owner, repo, files, onProgress) {
+  const pushed = [], failed = [];
+  for (const [fp, content] of Object.entries(files)) {
+    try {
+      await upsertFile(token, owner, repo, fp, content, `docs: update ${fp} via CDX [skip ci]`);
+      pushed.push(fp);
+      onProgress && onProgress(`Pushed: ${fp}`);
+    } catch (e) {
+      failed.push({ path: fp, error: e.message });
+      onProgress && onProgress(`Failed: ${fp} — ${e.message.split("\n")[0]}`);
+    }
+  }
+  return { pushed, failed };
+}
+
+async function fetchGitHubRepos(token) {
+  const all = [];
+  for (let page = 1; page <= 3; page++) {
+    const data = await githubRequest(token, "GET", `/user/repos?per_page=100&page=${page}&sort=updated`);
+    if (!data?.length) break;
+    all.push(...data);
+    if (data.length < 100) break;
+  }
+  return all;
+}
+
+// ─── Markdown section extractor ───────────────────────────────────────────────
+function extractSection(doc, heading) {
+  const re = new RegExp(`#\\s*${heading}([\\s\\S]*?)(?=\\n#\\s|$)`, "i");
+  const m  = doc.match(re);
+  return m ? m[1].trim() : null;
+}
+
+module.exports = {
+  runLocalPipeline,
+  runRemotePipeline,
+  pushToGitHub,
+  fetchGitHubRepos,
+  detectStack,
+  extractSection,
+  isSensitive,
+  IGNORE_DIRS, CODE_EXTS,
+};

package/lib/store.js

@@ -0,0 +1,83 @@
+"use strict";
+
+const fs   = require("fs");
+const path = require("path");
+const os   = require("os");
+
+const CDX_DIR    = path.join(os.homedir(), ".cdx");
+const CFG_PATH   = path.join(CDX_DIR, "config.json");
+const TOKEN_PATH = path.join(CDX_DIR, ".session");
+
+// ── Hardcoded Firebase config from google-services.json ───────────────────────
+// Project: mkchit | App: com.sas.mk
+const BUNDLED_FIREBASE_KEY = "xxxxxxxxxxxxxxxxxxxxxxxxxx";
+
+function ensureDir() {
+  if (!fs.existsSync(CDX_DIR)) fs.mkdirSync(CDX_DIR, { recursive: true, mode: 0o700 });
+}
+
+function load() {
+  ensureDir();
+  try {
+    var raw = fs.readFileSync(CFG_PATH, "utf8");
+    return JSON.parse(raw);
+  } catch { return {}; }
+}
+
+function save(cfg) {
+  ensureDir();
+  var tmp = CFG_PATH + ".tmp." + process.pid;
+  fs.writeFileSync(tmp, JSON.stringify(cfg, null, 2), { mode: 0o600 });
+  fs.renameSync(tmp, CFG_PATH);
+  try { fs.chmodSync(CFG_PATH, 0o600); } catch {}
+}
+
+function get(key) {
+  // firebaseApiKey always falls back to bundled key
+  if (key === "firebaseApiKey") {
+    var val = load()[key];
+    return val || BUNDLED_FIREBASE_KEY;
+  }
+  return load()[key];
+}
+
+function set(key, val) { var c = load(); c[key] = val; save(c); }
+function del(key)      { var c = load(); delete c[key]; save(c); }
+function clear()       { save({}); clearSession(); }
+
+// ── JWT session ───────────────────────────────────────────────────────────────
+function saveSession(jwt) {
+  ensureDir();
+  fs.writeFileSync(TOKEN_PATH, jwt, { mode: 0o600 });
+  try { fs.chmodSync(TOKEN_PATH, 0o600); } catch {}
+}
+
+function loadSession() {
+  try { return fs.readFileSync(TOKEN_PATH, "utf8").trim(); } catch { return null; }
+}
+
+function clearSession() {
+  try { fs.unlinkSync(TOKEN_PATH); } catch {}
+}
+
+function decodeJwt(token) {
+  try {
+    var payload = token.split(".")[1];
+    var padded  = payload + "=".repeat((4 - payload.length % 4) % 4);
+    return JSON.parse(Buffer.from(padded, "base64url").toString("utf8"));
+  } catch { return null; }
+}
+
+function sessionIsAlive(token) {
+  if (!token) return false;
+  var d = decodeJwt(token);
+  if (!d || !d.exp) return false;
+  return d.exp * 1000 > Date.now() + 60_000;
+}
+
+module.exports = {
+  load, save, get, set, del, clear,
+  saveSession, loadSession, clearSession,
+  decodeJwt, sessionIsAlive,
+  BUNDLED_FIREBASE_KEY,
+};