lens-engine 0.1.15 → 0.1.16

package/daemon.js

@@ -7717,6 +7717,11 @@ function openDb(customPath) {
   sqlite.pragma("foreign_keys = ON");
   const db = drizzle(sqlite, { schema: schema_exports });
   sqlite.exec(createTablesSql());
+  const cols = new Set(
+    sqlite.pragma("table_info(file_metadata)").map((c) => c.name)
+  );
+  if (!cols.has("sections")) sqlite.exec("ALTER TABLE file_metadata ADD COLUMN sections TEXT DEFAULT '[]'");
+  if (!cols.has("internals")) sqlite.exec("ALTER TABLE file_metadata ADD COLUMN internals TEXT DEFAULT '[]'");
   _db = db;
   _raw = sqlite;
   return db;
@@ -7781,6 +7786,8 @@ function createTablesSql() {
       exports TEXT DEFAULT '[]',
       imports TEXT DEFAULT '[]',
       docstring TEXT DEFAULT '',
+      sections TEXT DEFAULT '[]',
+      internals TEXT DEFAULT '[]',
       purpose TEXT DEFAULT '',
       purpose_hash TEXT DEFAULT '',
       updated_at TEXT NOT NULL DEFAULT (datetime('now')),
@@ -8238,13 +8245,59 @@ function extractDocstring(content, language) {
   if (!m) return "";
   return m[1]?.replace(/\s*\*\s*/g, " ").trim().slice(0, 200) ?? "";
 }
+function extractSections(content) {
+  const seen = /* @__PURE__ */ new Set();
+  const sections = [];
+  for (const re of [SECTION_SINGLE_RE, SECTION_BLOCK_RE]) {
+    const pattern = new RegExp(re.source, re.flags);
+    for (const m of content.matchAll(pattern)) {
+      const label = m[1]?.trim();
+      if (label && !seen.has(label)) {
+        seen.add(label);
+        sections.push(label);
+      }
+    }
+  }
+  return sections.slice(0, 15);
+}
+function extractInternals(content, language, exports) {
+  switch (language) {
+    case "typescript":
+    case "javascript":
+    case "tsx":
+    case "jsx":
+      break;
+    default:
+      return [];
+  }
+  const exportSet = new Set(exports);
+  const seen = /* @__PURE__ */ new Set();
+  const internals = [];
+  for (const line of content.split("\n")) {
+    if (line.trimStart().startsWith("export")) continue;
+    for (const re of [TS_INTERNAL_FN_RE, TS_INTERNAL_CONST_RE]) {
+      const pattern = new RegExp(re.source, re.flags);
+      for (const m of line.matchAll(pattern)) {
+        const name = m[1];
+        if (name && name.length >= 6 && !exportSet.has(name) && !seen.has(name)) {
+          seen.add(name);
+          internals.push(name);
+        }
+      }
+    }
+  }
+  return internals.slice(0, 20);
+}
 function extractFileMetadata(path, content, language) {
+  const exports = extractExports(content, language);
   return {
     path,
     language,
-    exports: extractExports(content, language),
+    exports,
     imports: extractImportSpecifiers(content, language),
-    docstring: extractDocstring(content, language)
+    docstring: extractDocstring(content, language),
+    sections: extractSections(content),
+    internals: extractInternals(content, language, exports)
   };
 }
 function extractAndPersistMetadata(db, repoId) {
@@ -8264,7 +8317,7 @@ function extractAndPersistMetadata(db, repoId) {
   let count = 0;
   for (const [path, { content, language }] of files) {
     const meta = extractFileMetadata(path, content, language);
-    metadataQueries.upsert(db, repoId, path, language, meta.exports, meta.imports, meta.docstring);
+    metadataQueries.upsert(db, repoId, path, language, meta.exports, meta.imports, meta.docstring, meta.sections, meta.internals);
     count++;
   }
   return count;
@@ -9063,7 +9116,7 @@ function buildTermWeights(words, metadata) {
   for (const w of words) {
     let df = 0;
     for (const f of metadata) {
-      const haystack = `${f.path} ${(f.exports ?? []).join(" ")} ${f.docstring ?? ""} ${f.purpose ?? ""}`.toLowerCase();
+      const haystack = `${f.path} ${(f.exports ?? []).join(" ")} ${f.docstring ?? ""} ${f.purpose ?? ""} ${(f.sections ?? []).join(" ")} ${(f.internals ?? []).join(" ")}`.toLowerCase();
       if (haystack.includes(w)) df++;
     }
     const idf = df > 0 ? Math.min(10, Math.max(1, Math.log(N / df))) : 10;
@@ -9083,6 +9136,8 @@ function interpretQuery(query, metadata, fileStats2, vocabClusters, indegrees, m
     const exportsLower = (f.exports ?? []).join(" ").toLowerCase();
     const docLower = (f.docstring ?? "").toLowerCase();
     const purposeLower = (f.purpose ?? "").toLowerCase();
+    const sectionsLower = (f.sections ?? []).join(" ").toLowerCase();
+    const internalsLower = (f.internals ?? []).join(" ").toLowerCase();
     const pathSegments = pathLower.split("/");
     const fileName = pathSegments[pathSegments.length - 1].replace(/\.[^.]+$/, "");
     const fileTokens = fileName.replace(/([a-z])([A-Z])/g, "$1 $2").replace(/[._-]/g, " ").toLowerCase().split(/\s+/).filter((t) => t.length >= 2);
@@ -9095,6 +9150,8 @@ function interpretQuery(query, metadata, fileStats2, vocabClusters, indegrees, m
       else if (pathTokenSet.has(w)) termScore += 2 * weight;
       if (exportsLower.includes(w)) termScore += 2 * weight;
       if (docLower.includes(w) || purposeLower.includes(w)) termScore += 1 * weight;
+      if (sectionsLower.includes(w)) termScore += 1 * weight;
+      if (internalsLower.includes(w)) termScore += 1.5 * weight;
       if (termScore > 0) matchedTerms++;
       score += termScore;
     }
@@ -9103,6 +9160,8 @@ function interpretQuery(query, metadata, fileStats2, vocabClusters, indegrees, m
       let termScore = 0;
       if (exportsLower.includes(w)) termScore += 2 * weight;
       if (docLower.includes(w) || purposeLower.includes(w)) termScore += 1 * weight;
+      if (sectionsLower.includes(w)) termScore += 1 * weight;
+      if (internalsLower.includes(w)) termScore += 1.5 * weight;
       if (termScore > 0) matchedTerms++;
       score += termScore;
     }
@@ -9404,7 +9463,7 @@ Context generation failed.`,
     };
   }
 }
-var __defProp2, __getOwnPropNames2, __esm2, __export2, schema_exports, uuid, now, updatedAt, repos, chunks, fileMetadata, fileImports, fileStats, fileCochanges, usageCounters, requestLogs, telemetryEvents, init_schema, queries_exports, repoQueries, chunkQueries, metadataQueries, importQueries, statsQueries, cochangeQueries, logQueries, usageQueries, telemetryQueries, init_queries, _db, _raw, execFileAsync, MAX_FILE_SIZE, BINARY_EXTENSIONS, DOCS_EXTENSIONS, LANG_MAP, DEFAULT_CHUNKING_PARAMS, TS_IMPORT_RE, PY_IMPORT_RE, GO_IMPORT_RE, RUST_USE_RE, TS_EXTENSIONS, PY_EXTENSIONS, TS_EXPORT_RE, PY_EXPORT_RE, GO_EXPORT_RE, RUST_EXPORT_RE, CSHARP_EXPORT_RE, JAVA_EXPORT_RE, JSDOC_RE, PY_DOCSTRING_RE, CSHARP_DOC_RE, GO_PKG_RE, TERM_BATCH_SIZE, SIMILARITY_THRESHOLD, MAX_TERMS, MAX_CLUSTERS, MAX_CLUSTER_SIZE, STOPWORDS, execFileAsync2, MAX_COMMITS, MAX_FILES_PER_COMMIT, RECENT_DAYS, _enabled, MAX_CHUNKS_PER_REPO, locks, MAX_API_CALLS, POOL_SIZE, MAX_BATCH_TOKENS, CHARS_PER_TOKEN, PURPOSE_BATCH_LIMIT, PURPOSE_CONCURRENCY, watchers, debounceTimers, IGNORED, DEBOUNCE_MS, STOPWORDS2, NOISE_EXTENSIONS, NOISE_PATHS, CONCEPT_SYNONYMS, CACHE_TTL, CACHE_MAX, cache;
+var __defProp2, __getOwnPropNames2, __esm2, __export2, schema_exports, uuid, now, updatedAt, repos, chunks, fileMetadata, fileImports, fileStats, fileCochanges, usageCounters, requestLogs, telemetryEvents, init_schema, queries_exports, repoQueries, chunkQueries, metadataQueries, importQueries, statsQueries, cochangeQueries, logQueries, usageQueries, telemetryQueries, init_queries, _db, _raw, execFileAsync, MAX_FILE_SIZE, BINARY_EXTENSIONS, DOCS_EXTENSIONS, LANG_MAP, DEFAULT_CHUNKING_PARAMS, TS_IMPORT_RE, PY_IMPORT_RE, GO_IMPORT_RE, RUST_USE_RE, TS_EXTENSIONS, PY_EXTENSIONS, TS_EXPORT_RE, PY_EXPORT_RE, GO_EXPORT_RE, RUST_EXPORT_RE, CSHARP_EXPORT_RE, JAVA_EXPORT_RE, JSDOC_RE, PY_DOCSTRING_RE, CSHARP_DOC_RE, GO_PKG_RE, SECTION_SINGLE_RE, SECTION_BLOCK_RE, TS_INTERNAL_FN_RE, TS_INTERNAL_CONST_RE, TERM_BATCH_SIZE, SIMILARITY_THRESHOLD, MAX_TERMS, MAX_CLUSTERS, MAX_CLUSTER_SIZE, STOPWORDS, execFileAsync2, MAX_COMMITS, MAX_FILES_PER_COMMIT, RECENT_DAYS, _enabled, MAX_CHUNKS_PER_REPO, locks, MAX_API_CALLS, POOL_SIZE, MAX_BATCH_TOKENS, CHARS_PER_TOKEN, PURPOSE_BATCH_LIMIT, PURPOSE_CONCURRENCY, watchers, debounceTimers, IGNORED, DEBOUNCE_MS, STOPWORDS2, NOISE_EXTENSIONS, NOISE_PATHS, CONCEPT_SYNONYMS, CACHE_TTL, CACHE_MAX, cache;
 var init_dist = __esm({
   "packages/engine/dist/index.js"() {
     "use strict";
@@ -9493,6 +9552,8 @@ var init_dist = __esm({
             exports: text("exports").default("[]"),
             imports: text("imports").default("[]"),
             docstring: text("docstring").default(""),
+            sections: text("sections").default("[]"),
+            internals: text("internals").default("[]"),
             purpose: text("purpose").default(""),
             purpose_hash: text("purpose_hash").default(""),
             updated_at: updatedAt()
@@ -9785,7 +9846,7 @@ var init_dist = __esm({
           }
         };
         metadataQueries = {
-          upsert(db, repoId, path, language, exports, imports, docstring) {
+          upsert(db, repoId, path, language, exports, imports, docstring, sections, internals) {
             db.insert(fileMetadata).values({
               id: randomUUID(),
               repo_id: repoId,
@@ -9793,7 +9854,9 @@ var init_dist = __esm({
               language,
               exports: JSON.stringify(exports),
               imports: JSON.stringify(imports),
-              docstring
+              docstring,
+              sections: JSON.stringify(sections),
+              internals: JSON.stringify(internals)
             }).onConflictDoUpdate({
               target: [fileMetadata.repo_id, fileMetadata.path],
               set: {
@@ -9801,6 +9864,8 @@ var init_dist = __esm({
                 exports: JSON.stringify(exports),
                 imports: JSON.stringify(imports),
                 docstring,
+                sections: JSON.stringify(sections),
+                internals: JSON.stringify(internals),
                 updated_at: sql`datetime('now')`
               }
             }).run();
@@ -9811,11 +9876,15 @@ var init_dist = __esm({
               language: fileMetadata.language,
               exports: fileMetadata.exports,
               docstring: fileMetadata.docstring,
+              sections: fileMetadata.sections,
+              internals: fileMetadata.internals,
               purpose: fileMetadata.purpose
             }).from(fileMetadata).where(eq(fileMetadata.repo_id, repoId)).orderBy(fileMetadata.path).all().map((r) => ({
               ...r,
               exports: jsonParse(r.exports, []),
               docstring: r.docstring ?? "",
+              sections: jsonParse(r.sections, []),
+              internals: jsonParse(r.internals, []),
               purpose: r.purpose ?? ""
             }));
           },
@@ -10184,6 +10253,10 @@ var init_dist = __esm({
     PY_DOCSTRING_RE = /^(?:["']{3})([\s\S]*?)(?:["']{3})/m;
     CSHARP_DOC_RE = /^(?:\s*\/\/\/\s*(.*))+/m;
     GO_PKG_RE = /^\/\/\s*Package\s+\w+\s+(.*)/m;
+    SECTION_SINGLE_RE = /^(?:\/\/|#)\s*[-=]{3,}\s*(.+?)\s*[-=]{3,}\s*$/gm;
+    SECTION_BLOCK_RE = /^\/\*\s*[-=]{3,}\s*(.+?)\s*[-=]{3,}\s*\*\/$/gm;
+    TS_INTERNAL_FN_RE = /^(?:async\s+)?function\s+(\w+)/gm;
+    TS_INTERNAL_CONST_RE = /^(?:const|let)\s+(\w+)\s*=/gm;
     init_queries();
     init_queries();
     TERM_BATCH_SIZE = 32;
@@ -34525,7 +34598,7 @@ function createApp(db, dashboardDist, initialCaps, initialPlanData) {
     const duration3 = Math.round(performance.now() - start);
     const path = new URL(c.req.url).pathname;
     const source = deriveSource(c.req.raw, path);
-    if (path.startsWith("/dashboard/") && !path.startsWith("/api/")) return;
+    if (path.startsWith("/dashboard/") || path.startsWith("/api/dashboard/")) return;
     if (path === "/health") return;
     try {
       logQueries.insert(db, c.req.method, path, c.res.status, duration3, source);

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "lens-engine",
-	"version": "0.1.15",
+	"version": "0.1.16",
 	"description": "LENS — Local-first repo context engine for AI agents",
 	"type": "module",
 	"bin": {