@folterung/project-memory 0.1.0

package/packages/cli/src/phase2/deep-index.ts

@@ -0,0 +1,120 @@
+import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs";
+import { join } from "node:path";
+import chalk from "chalk";
+import { loadConfig } from "../config/load.js";
+import { chunkFile } from "../chunking/chunker.js";
+import type { Chunk } from "../chunking/types.js";
+import { embedBatch } from "../embedding/stub.js";
+import { createQdrantClient, ensureCollection, upsertPoints } from "../qdrant/upsert.js";
+import { listInScopeFiles } from "../scope/allowlist.js";
+import { createIgnoreFilter } from "../scope/ignore.js";
+import { toRelativePath } from "../scope/allowlist.js";
+import { DEEP_INDEX_STATE_FILE } from "../config/types.js";
+
+const BATCH_SIZE = 8;
+const BATCH_DELAY_MS = 200;
+const PHASE1_KINDS = new Set(["system_map", "module_summary", "entrypoint"]);
+
+interface DeepIndexState {
+  embeddedIds: string[];
+  total: number;
+  lastUpdated: string;
+}
+
+function loadDeepIndexState(cwd: string): DeepIndexState {
+  const path = join(cwd, DEEP_INDEX_STATE_FILE);
+  if (!existsSync(path)) return { embeddedIds: [], total: 0, lastUpdated: "" };
+  try {
+    const raw = readFileSync(path, "utf8");
+    const data = JSON.parse(raw) as { embeddedIds?: string[]; total?: number; lastUpdated?: string };
+    return {
+      embeddedIds: data.embeddedIds ?? [],
+      total: data.total ?? 0,
+      lastUpdated: data.lastUpdated ?? "",
+    };
+  } catch {
+    return { embeddedIds: [], total: 0, lastUpdated: "" };
+  }
+}
+
+function saveDeepIndexState(cwd: string, state: DeepIndexState): void {
+  const stateDir = join(cwd, ".mem", "state");
+  if (!existsSync(stateDir)) mkdirSync(stateDir, { recursive: true });
+  writeFileSync(join(cwd, DEEP_INDEX_STATE_FILE), JSON.stringify(state, null, 2), "utf8");
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+export async function runDeepIndex(cwd: string): Promise<void> {
+  const config = loadConfig(cwd);
+  const isIgnored = createIgnoreFilter(cwd, config.indexing?.exclude ?? []);
+  const inScopePaths = listInScopeFiles(cwd, config);
+  const eligiblePaths: string[] = [];
+  for (const abs of inScopePaths) {
+    const rel = toRelativePath(cwd, abs);
+    if (!isIgnored(rel)) eligiblePaths.push(abs);
+  }
+
+  const allChunks: Chunk[] = [];
+  for (const abs of eligiblePaths) {
+    try {
+      const content = readFileSync(abs, "utf8");
+      const chunks = chunkFile(toRelativePath(cwd, abs), content);
+      allChunks.push(...chunks);
+    } catch {
+      // skip
+    }
+  }
+
+  let state = loadDeepIndexState(cwd);
+  if (!state.embeddedIds.length && state.total === 0) {
+    state = { embeddedIds: [], total: allChunks.length, lastUpdated: "" };
+  }
+  const embeddedSet = new Set(state.embeddedIds);
+  const remaining = allChunks.filter((c) => !embeddedSet.has(c.id));
+
+  if (remaining.length === 0) {
+    console.log(chalk.green("Deep index already complete."));
+    return;
+  }
+
+  const host = config.qdrant?.host ?? "127.0.0.1";
+  const port = config.qdrant?.port ?? 6333;
+  const collection = config.qdrant?.collection ?? "project_memory";
+  const client = createQdrantClient(host, port);
+  await ensureCollection(client, collection);
+
+  let embeddedCount = state.embeddedIds.length;
+  const total = allChunks.length;
+
+  for (let i = 0; i < remaining.length; i += BATCH_SIZE) {
+    const batch = remaining.slice(i, i + BATCH_SIZE);
+    const texts = batch.map((c) => c.text.slice(0, 8000));
+    const vectors = embedBatch(texts);
+    const points = batch.map((c, j) => ({
+      id: c.id,
+      vector: vectors[j],
+      payload: {
+        kind: "chunk",
+        path: c.metadata.path,
+        text: texts[j],
+      },
+    }));
+    await upsertPoints(client, collection, points);
+    embeddedCount += batch.length;
+    const newEmbeddedIds = [...state.embeddedIds, ...batch.map((c) => c.id)];
+    state.embeddedIds = newEmbeddedIds;
+    state.total = total;
+    state.lastUpdated = new Date().toISOString();
+    saveDeepIndexState(cwd, state);
+
+    const percent = total > 0 ? Math.round((embeddedCount / total) * 100) : 0;
+    console.log(chalk.dim(`  Deep index: ${embeddedCount}/${total} (${percent}%)`));
+
+    await sleep(BATCH_DELAY_MS);
+  }
+
+  console.log(chalk.green("✓"), "Deep index complete:", embeddedCount, "chunks");
+}

package/packages/cli/src/qdrant/upsert.ts

@@ -0,0 +1,45 @@
+import { QdrantClient } from "@qdrant/js-client-rest";
+import { getVectorSize } from "../embedding/stub.js";
+
+const DEFAULT_COLLECTION = "project_memory";
+
+export function createQdrantClient(host: string = "127.0.0.1", port: number = 6333): QdrantClient {
+  return new QdrantClient({ url: `http://${host}:${port}` });
+}
+
+export async function ensureCollection(
+  client: QdrantClient,
+  collectionName: string = DEFAULT_COLLECTION
+): Promise<void> {
+  const size = getVectorSize();
+  try {
+    await client.getCollection(collectionName);
+  } catch {
+    await client.createCollection(collectionName, {
+      vectors: { size, distance: "Cosine" },
+    });
+  }
+}
+
+export interface PointPayload {
+  kind: string;
+  path?: string;
+  text: string;
+  [key: string]: unknown;
+}
+
+export async function upsertPoints(
+  client: QdrantClient,
+  collectionName: string,
+  points: { id: string; vector: number[]; payload: PointPayload }[]
+): Promise<void> {
+  if (points.length === 0) return;
+  await client.upsert(collectionName, {
+    wait: true,
+    points: points.map((p) => ({
+      id: p.id,
+      vector: p.vector,
+      payload: p.payload,
+    })),
+  });
+}

package/packages/cli/src/scope/allowlist.test.ts

@@ -0,0 +1,25 @@
+import { strict as assert } from "node:assert";
+import { describe, it } from "node:test";
+import { toRelativePath } from "./allowlist.js";
+
+describe("scope allowlist", () => {
+  it("toRelativePath strips cwd and normalizes slashes", () => {
+    const cwd = "/home/repo";
+    const abs = "/home/repo/src/foo.ts";
+    assert.strictEqual(toRelativePath(cwd, abs), "src/foo.ts");
+  });
+
+  it("toRelativePath handles trailing slash in cwd", () => {
+    const cwd = "/home/repo/";
+    const abs = "/home/repo/src/foo.ts";
+    assert.strictEqual(toRelativePath(cwd, abs), "src/foo.ts");
+  });
+
+  it("toRelativePath uses forward slashes", () => {
+    const cwd = "C:\\repo";
+    const abs = "C:\\repo\\src\\foo.ts";
+    const rel = toRelativePath(cwd, abs);
+    assert.ok(!rel.includes("\\"));
+    assert.strictEqual(rel, "src/foo.ts");
+  });
+});

package/packages/cli/src/scope/allowlist.ts

@@ -0,0 +1,38 @@
+import fg from "fast-glob";
+import { join } from "node:path";
+import type { MemConfig } from "../config/types.js";
+
+/**
+ * Resolve include patterns to a list of absolute paths that are in scope.
+ * Apply BEFORE any directory traversal. Only these paths are eligible.
+ * If indexing.include is absent, entire repo root is eligible (we still need to list files).
+ */
+export function listInScopeFiles(cwd: string, config: MemConfig): string[] {
+  const include = config.indexing?.include;
+  if (!include?.length) {
+    const all = fg.sync("**/*", {
+      cwd,
+      dot: true,
+      onlyFiles: true,
+      absolute: true,
+      suppressErrors: true,
+    });
+    return all.map((p) => p.startsWith(cwd) ? p : join(cwd, p));
+  }
+  const files = fg.sync(include, {
+    cwd,
+    dot: true,
+    onlyFiles: true,
+    absolute: true,
+    suppressErrors: true,
+  });
+  return files.map((p) => (p.startsWith(cwd) ? p : join(cwd, p)));
+}
+
+/**
+ * Given an absolute path, return the relative path from cwd for ignore matching.
+ */
+export function toRelativePath(cwd: string, absolutePath: string): string {
+  const rel = absolutePath.slice(cwd.length).replace(/^[/\\]+/, "");
+  return rel.replace(/\\/g, "/");
+}

package/packages/cli/src/scope/ignore.test.ts

@@ -0,0 +1,71 @@
+import { strict as assert } from "node:assert";
+import { describe, it } from "node:test";
+import { mkdtempSync, writeFileSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { createIgnoreFilter, getBuiltinIgnores } from "./ignore.js";
+
+describe("scope ignore", () => {
+  it("getBuiltinIgnores includes node_modules and .env", () => {
+    const builtin = getBuiltinIgnores();
+    assert.ok(builtin.some((p) => p.includes("node_modules")), "builtin should ignore node_modules");
+    assert.ok(builtin.includes(".env"));
+  });
+
+  it("createIgnoreFilter ignores node_modules", () => {
+    const cwd = mkdtempSync(join(tmpdir(), "mem-ignore-"));
+    try {
+      const isIgnored = createIgnoreFilter(cwd, []);
+      assert.strictEqual(isIgnored("node_modules/foo/bar.js"), true);
+      assert.strictEqual(isIgnored("src/foo.js"), false);
+    } finally {
+      rmSync(cwd, { recursive: true, force: true });
+    }
+  });
+
+  it("createIgnoreFilter ignores .env", () => {
+    const cwd = mkdtempSync(join(tmpdir(), "mem-ignore-"));
+    try {
+      const isIgnored = createIgnoreFilter(cwd, []);
+      assert.strictEqual(isIgnored(".env"), true);
+      assert.strictEqual(isIgnored(".env.local"), true);
+    } finally {
+      rmSync(cwd, { recursive: true, force: true });
+    }
+  });
+
+  it("createIgnoreFilter respects .memignore when present", () => {
+    const cwd = mkdtempSync(join(tmpdir(), "mem-ignore-"));
+    writeFileSync(join(cwd, ".memignore"), "*.skip\n", "utf8");
+    try {
+      const isIgnored = createIgnoreFilter(cwd, []);
+      assert.strictEqual(isIgnored("foo.skip"), true);
+      assert.strictEqual(isIgnored("foo.ts"), false);
+    } finally {
+      rmSync(cwd, { recursive: true, force: true });
+    }
+  });
+
+  it("createIgnoreFilter negation re-includes", () => {
+    const cwd = mkdtempSync(join(tmpdir(), "mem-ignore-"));
+    writeFileSync(join(cwd, ".memignore"), "*.skip\n!important.skip\n", "utf8");
+    try {
+      const isIgnored = createIgnoreFilter(cwd, []);
+      assert.strictEqual(isIgnored("foo.skip"), true);
+      assert.strictEqual(isIgnored("important.skip"), false);
+    } finally {
+      rmSync(cwd, { recursive: true, force: true });
+    }
+  });
+
+  it("createIgnoreFilter respects config exclude", () => {
+    const cwd = mkdtempSync(join(tmpdir(), "mem-ignore-"));
+    try {
+      const isIgnored = createIgnoreFilter(cwd, ["**/excluded/**"]);
+      assert.strictEqual(isIgnored("src/excluded/bar.js"), true);
+      assert.strictEqual(isIgnored("src/other/bar.js"), false);
+    } finally {
+      rmSync(cwd, { recursive: true, force: true });
+    }
+  });
+});

package/packages/cli/src/scope/ignore.ts

@@ -0,0 +1,86 @@
+import { readFileSync, existsSync } from "node:fs";
+import { join } from "node:path";
+import { Minimatch } from "minimatch";
+import { MEMIGNORE_FILE } from "../config/types.js";
+
+const BUILTIN_IGNORES = [
+  "node_modules/**",
+  "vendor/**",
+  "dist/**",
+  "build/**",
+  ".next/**",
+  ".cache/**",
+  "**/*.png",
+  "**/*.jpg",
+  "**/*.jpeg",
+  "**/*.gif",
+  "**/*.mp4",
+  "**/*.zip",
+  ".env",
+  ".env.*",
+  "**/*.pem",
+  "**/*.key",
+  "**/id_rsa",
+  ".mem/",
+];
+
+function parseIgnoreLines(content: string): { pattern: string; negated: boolean }[] {
+  const out: { pattern: string; negated: boolean }[] = [];
+  for (const line of content.split(/\r?\n/)) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#")) continue;
+    const negated = trimmed.startsWith("!");
+    const pattern = negated ? trimmed.slice(1).trim() : trimmed;
+    if (pattern) out.push({ pattern, negated });
+  }
+  return out;
+}
+
+function loadMemignore(cwd: string): { pattern: string; negated: boolean }[] {
+  const path = join(cwd, MEMIGNORE_FILE);
+  if (!existsSync(path)) return [];
+  const content = readFileSync(path, "utf8");
+  return parseIgnoreLines(content);
+}
+
+function loadConfigExclude(cwd: string, excludePatterns: string[]): { pattern: string; negated: boolean }[] {
+  if (!excludePatterns?.length) return [];
+  return excludePatterns.map((pattern) => ({ pattern, negated: false }));
+}
+
+function buildMatcher(pattern: string): (p: string) => boolean {
+  const m = new Minimatch(pattern, { dot: true, matchBase: !pattern.includes("/") });
+  return (p: string) => m.match(p);
+}
+
+export function createIgnoreFilter(
+  cwd: string,
+  configExclude: string[] = []
+): (relativePath: string) => boolean {
+  const builtinRules: { pattern: string; negated: boolean }[] = BUILTIN_IGNORES.map((pattern) => ({
+    pattern,
+    negated: false,
+  }));
+  const memignoreRules = loadMemignore(cwd);
+  const configRules = loadConfigExclude(cwd, configExclude);
+  const ordered: { match: (p: string) => boolean; negated: boolean }[] = [
+    ...builtinRules.map((r) => ({ match: buildMatcher(r.pattern), negated: r.negated })),
+    ...memignoreRules.map((r) => ({ match: buildMatcher(r.pattern), negated: r.negated })),
+    ...configRules.map((r) => ({ match: buildMatcher(r.pattern), negated: r.negated })),
+  ];
+
+  return function isIgnored(relativePath: string): boolean {
+    const normalized = relativePath.replace(/\\/g, "/");
+    let ignored = false;
+    for (const { match, negated } of ordered) {
+      if (match(normalized)) {
+        ignored = !negated;
+      }
+    }
+    return ignored;
+  };
+}
+
+export function getBuiltinIgnores(): string[] {
+  return [...BUILTIN_IGNORES];
+}

package/packages/cli/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "outDir": "dist",
+    "rootDir": "src",
+    "declaration": true,
+    "sourceMap": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}

package/packages/server/coverage/lcov-report/base.css

@@ -0,0 +1,224 @@
+body, html {
+  margin:0; padding: 0;
+  height: 100%;
+}
+body {
+    font-family: Helvetica Neue, Helvetica, Arial;
+    font-size: 14px;
+    color:#333;
+}
+.small { font-size: 12px; }
+*, *:after, *:before {
+  -webkit-box-sizing:border-box;
+     -moz-box-sizing:border-box;
+          box-sizing:border-box;
+  }
+h1 { font-size: 20px; margin: 0;}
+h2 { font-size: 14px; }
+pre {
+    font: 12px/1.4 Consolas, "Liberation Mono", Menlo, Courier, monospace;
+    margin: 0;
+    padding: 0;
+    -moz-tab-size: 2;
+    -o-tab-size:  2;
+    tab-size: 2;
+}
+a { color:#0074D9; text-decoration:none; }
+a:hover { text-decoration:underline; }
+.strong { font-weight: bold; }
+.space-top1 { padding: 10px 0 0 0; }
+.pad2y { padding: 20px 0; }
+.pad1y { padding: 10px 0; }
+.pad2x { padding: 0 20px; }
+.pad2 { padding: 20px; }
+.pad1 { padding: 10px; }
+.space-left2 { padding-left:55px; }
+.space-right2 { padding-right:20px; }
+.center { text-align:center; }
+.clearfix { display:block; }
+.clearfix:after {
+  content:'';
+  display:block;
+  height:0;
+  clear:both;
+  visibility:hidden;
+  }
+.fl { float: left; }
+@media only screen and (max-width:640px) {
+  .col3 { width:100%; max-width:100%; }
+  .hide-mobile { display:none!important; }
+}
+
+.quiet {
+  color: #7f7f7f;
+  color: rgba(0,0,0,0.5);
+}
+.quiet a { opacity: 0.7; }
+
+.fraction {
+  font-family: Consolas, 'Liberation Mono', Menlo, Courier, monospace;
+  font-size: 10px;
+  color: #555;
+  background: #E8E8E8;
+  padding: 4px 5px;
+  border-radius: 3px;
+  vertical-align: middle;
+}
+
+div.path a:link, div.path a:visited { color: #333; }
+table.coverage {
+  border-collapse: collapse;
+  margin: 10px 0 0 0;
+  padding: 0;
+}
+
+table.coverage td {
+  margin: 0;
+  padding: 0;
+  vertical-align: top;
+}
+table.coverage td.line-count {
+    text-align: right;
+    padding: 0 5px 0 20px;
+}
+table.coverage td.line-coverage {
+    text-align: right;
+    padding-right: 10px;
+    min-width:20px;
+}
+
+table.coverage td span.cline-any {
+    display: inline-block;
+    padding: 0 5px;
+    width: 100%;
+}
+.missing-if-branch {
+    display: inline-block;
+    margin-right: 5px;
+    border-radius: 3px;
+    position: relative;
+    padding: 0 4px;
+    background: #333;
+    color: yellow;
+}
+
+.skip-if-branch {
+    display: none;
+    margin-right: 10px;
+    position: relative;
+    padding: 0 4px;
+    background: #ccc;
+    color: white;
+}
+.missing-if-branch .typ, .skip-if-branch .typ {
+    color: inherit !important;
+}
+.coverage-summary {
+  border-collapse: collapse;
+  width: 100%;
+}
+.coverage-summary tr { border-bottom: 1px solid #bbb; }
+.keyline-all { border: 1px solid #ddd; }
+.coverage-summary td, .coverage-summary th { padding: 10px; }
+.coverage-summary tbody { border: 1px solid #bbb; }
+.coverage-summary td { border-right: 1px solid #bbb; }
+.coverage-summary td:last-child { border-right: none; }
+.coverage-summary th {
+  text-align: left;
+  font-weight: normal;
+  white-space: nowrap;
+}
+.coverage-summary th.file { border-right: none !important; }
+.coverage-summary th.pct { }
+.coverage-summary th.pic,
+.coverage-summary th.abs,
+.coverage-summary td.pct,
+.coverage-summary td.abs { text-align: right; }
+.coverage-summary td.file { white-space: nowrap;  }
+.coverage-summary td.pic { min-width: 120px !important;  }
+.coverage-summary tfoot td { }
+
+.coverage-summary .sorter {
+    height: 10px;
+    width: 7px;
+    display: inline-block;
+    margin-left: 0.5em;
+    background: url(sort-arrow-sprite.png) no-repeat scroll 0 0 transparent;
+}
+.coverage-summary .sorted .sorter {
+    background-position: 0 -20px;
+}
+.coverage-summary .sorted-desc .sorter {
+    background-position: 0 -10px;
+}
+.status-line {  height: 10px; }
+/* yellow */
+.cbranch-no { background: yellow !important; color: #111; }
+/* dark red */
+.red.solid, .status-line.low, .low .cover-fill { background:#C21F39 }
+.low .chart { border:1px solid #C21F39 }
+.highlighted,
+.highlighted .cstat-no, .highlighted .fstat-no, .highlighted .cbranch-no{
+  background: #C21F39 !important;
+}
+/* medium red */
+.cstat-no, .fstat-no, .cbranch-no, .cbranch-no { background:#F6C6CE }
+/* light red */
+.low, .cline-no { background:#FCE1E5 }
+/* light green */
+.high, .cline-yes { background:rgb(230,245,208) }
+/* medium green */
+.cstat-yes { background:rgb(161,215,106) }
+/* dark green */
+.status-line.high, .high .cover-fill { background:rgb(77,146,33) }
+.high .chart { border:1px solid rgb(77,146,33) }
+/* dark yellow (gold) */
+.status-line.medium, .medium .cover-fill { background: #f9cd0b; }
+.medium .chart { border:1px solid #f9cd0b; }
+/* light yellow */
+.medium { background: #fff4c2; }
+
+.cstat-skip { background: #ddd; color: #111; }
+.fstat-skip { background: #ddd; color: #111 !important; }
+.cbranch-skip { background: #ddd !important; color: #111; }
+
+span.cline-neutral { background: #eaeaea; }
+
+.coverage-summary td.empty {
+    opacity: .5;
+    padding-top: 4px;
+    padding-bottom: 4px;
+    line-height: 1;
+    color: #888;
+}
+
+.cover-fill, .cover-empty {
+  display:inline-block;
+  height: 12px;
+}
+.chart {
+  line-height: 0;
+}
+.cover-empty {
+    background: white;
+}
+.cover-full {
+    border-right: none !important;
+}
+pre.prettyprint {
+    border: none !important;
+    padding: 0 !important;
+    margin: 0 !important;
+}
+.com { color: #999 !important; }
+.ignore-none { color: #999; font-weight: normal; }
+
+.wrapper {
+  min-height: 100%;
+  height: auto !important;
+  height: 100%;
+  margin: 0 auto -48px;
+}
+.footer, .push {
+  height: 48px;
+}