llmnav 0.5.1

package/src/spec.js

@@ -0,0 +1,237 @@
+export const PACKAGE_VERSION = "0.5.1";
+export const SPEC_VERSION = "1";
+
+export const SCOPES = Object.freeze(["file", "module", "symbol"]);
+export const STABILITIES = Object.freeze(["architecture", "contract", "implementation"]);
+
+export const KEY_ORDER = Object.freeze([
+  "id",
+  "role",
+  "owns",
+  "excludes",
+  "search",
+  "invariant",
+  "effect",
+  "risk",
+  "rel",
+  "stability",
+]);
+
+export const REQUIRED_KEYS = Object.freeze(["id", "role", "stability"]);
+export const REPEATABLE_KEYS = Object.freeze(["invariant", "rel"]);
+export const LIST_KEYS = Object.freeze(["owns", "excludes", "search", "effect", "risk"]);
+export const SCALAR_KEYS = Object.freeze(["id", "role", "stability"]);
+export const ALLOWED_KEYS = Object.freeze([...KEY_ORDER]);
+
+export const EFFECT_KINDS_WITH_ARGUMENT = Object.freeze([
+  "db.read",
+  "db.write",
+  "cache.read",
+  "cache.write",
+  "event.emit",
+  "event.consume",
+  "net.call",
+  "lock.acquire",
+  "cookie.write",
+  "auth.check",
+]);
+
+export const EFFECT_KINDS_WITHOUT_ARGUMENT = Object.freeze([
+  "fs.read",
+  "fs.write",
+  "process.spawn",
+  "clock.read",
+  "random.read",
+]);
+
+export const EFFECT_KINDS = Object.freeze([
+  ...EFFECT_KINDS_WITH_ARGUMENT,
+  ...EFFECT_KINDS_WITHOUT_ARGUMENT,
+]);
+
+export const RISK_KINDS = Object.freeze([
+  "auth",
+  "money",
+  "privacy",
+  "concurrency",
+  "migration",
+  "availability",
+  "performance",
+]);
+
+export const STRICT_RISKS = Object.freeze(["auth", "money", "privacy"]);
+
+export const RELATION_KINDS = Object.freeze([
+  "policy",
+  "workflow",
+  "fallback",
+  "mirror",
+  "migration",
+  "test",
+  "replaces",
+  "deprecated-by",
+  "cross-repo",
+]);
+
+export const FORBIDDEN_STRUCTURE_RELATIONS = Object.freeze([
+  "calls",
+  "imports",
+  "references",
+  "implements",
+  "exports",
+  "overrides",
+]);
+
+export const FORBIDDEN_VOLATILE_KEYS = Object.freeze([
+  "path",
+  "line",
+  "span",
+  "commit",
+  "updated_at",
+  "updated-at",
+  "owner",
+  "callers",
+  "callees",
+  "imports",
+  "references",
+  "implementation_count",
+  "implementation-count",
+  "test_status",
+  "test-status",
+  "current_signature",
+  "current-signature",
+]);
+
+export const DEFAULT_INCLUDE_EXTENSIONS = Object.freeze([
+  ".ts",
+  ".tsx",
+  ".js",
+  ".jsx",
+  ".mjs",
+  ".cjs",
+  ".go",
+  ".rs",
+  ".py",
+  ".java",
+  ".kt",
+  ".kts",
+  ".c",
+  ".cc",
+  ".cpp",
+  ".h",
+  ".hpp",
+  ".cs",
+  ".swift",
+  ".dart",
+  ".php",
+  ".rb",
+  ".sh",
+  ".bash",
+  ".zsh",
+  ".sql",
+  ".svelte",
+  ".astro",
+  ".vue",
+]);
+
+export const DEFAULT_EXCLUDED_DIRECTORIES = Object.freeze([
+  ".git",
+  ".hg",
+  ".svn",
+  ".idea",
+  ".vscode",
+  "node_modules",
+  "vendor",
+  "dist",
+  "build",
+  "coverage",
+  ".next",
+  ".nuxt",
+  ".svelte-kit",
+  ".astro",
+  ".turbo",
+  ".cache",
+  "target",
+  "bin",
+  "obj",
+]);
+
+export const DEFAULT_GENERIC_SEARCH_TERMS = Object.freeze([
+  "service",
+  "manager",
+  "handler",
+  "helper",
+  "utility",
+  "util",
+  "data",
+  "process",
+  "logic",
+  "common",
+  "shared",
+]);
+
+export const DEFAULT_VAGUE_ROLE_WORDS = Object.freeze([
+  "handle",
+  "handles",
+  "manage",
+  "manages",
+  "process",
+  "processes",
+  "utility",
+  "helper",
+  "service",
+]);
+
+export const ID_PATTERN = /^[a-z][a-z0-9]*(?:\.[a-z][a-z0-9-]*){1,5}$/u;
+export const CROSS_REPO_ID_PATTERN = /^[a-z][a-z0-9-]*\/[a-z][a-z0-9]*(?:\.[a-z][a-z0-9-]*){1,5}$/u;
+
+export const DEFAULT_CONFIG = Object.freeze({
+  $schema: "./schema/config.schema.json",
+  version: 1,
+  repositoryId: "change-me",
+  sourceRoots: ["."],
+  includeExtensions: [...DEFAULT_INCLUDE_EXTENSIONS],
+  excludeDirectories: [...DEFAULT_EXCLUDED_DIRECTORIES],
+  excludeFiles: ["*.min.js", "*.bundle.js", "*.generated.*", "*.gen.*"],
+  coverageRules: [],
+  graph: {
+    indexFiles: [],
+  },
+  lint: {
+    maxRoleLength: 180,
+    maxSearchTerms: 6,
+    minSearchTerms: 2,
+    maxInvariants: 4,
+    maxEffects: 6,
+    maxRelations: 6,
+    maxBlockBytes: {
+      file: 400,
+      module: 1200,
+      symbol: 900,
+    },
+    maxSemanticRatio: 0.015,
+    minimumSourceBytesForRatio: 50000,
+    searchTermSaturation: 0.05,
+    minimumCardsForSaturation: 20,
+    genericSearchTerms: [...DEFAULT_GENERIC_SEARCH_TERMS],
+    vagueRoleWords: [...DEFAULT_VAGUE_ROLE_WORDS],
+    strictRisks: [...STRICT_RISKS],
+    additionalEffects: [],
+    additionalRisks: [],
+    additionalRelations: [],
+    requireCanonicalOrder: true,
+    requireCanonicalFormatting: true,
+  },
+  generation: {
+    cacheDirectory: ".llmnav/cache",
+    moduleDepth: 2,
+    searchShardSize: 0,
+    repositoryCatalogStabilities: ["architecture"],
+    moduleCatalogStabilities: ["architecture", "contract"],
+  },
+  evaluation: {
+    queryFile: ".llmnav/eval/queries.jsonl",
+    minimumRecallAt1: 0.75,
+    minimumRecallAt5: 0.9,
+  },
+});

package/src/tokenizer.js

@@ -0,0 +1,37 @@
+/* llmnav/1 module
+id=llmnav.search.tokenize
+role=Normalize task text and emit deterministic word and CJK n-gram tokens for semantic retrieval.
+owns=query normalization|CJK n-grams|tokenizer version
+excludes=card ranking|inverted-index persistence
+search=search tokenizer|CJK ngram|query normalization
+rel=workflow>llmnav.search.query
+stability=architecture
+*/
+
+export const TOKENIZER_VERSION = 1;
+
+export function normalizeSearchText(value) {
+  return String(value ?? "").normalize("NFKC").toLocaleLowerCase("en-US").trim();
+}
+
+export function tokenize(value) {
+  const normalized = normalizeSearchText(value);
+  const base = normalized.match(/[\p{L}\p{N}]+/gu) ?? [];
+  const tokens = [...base];
+  for (const token of base) {
+    if (containsCjk(token) && [...token].length >= 3) {
+      const characters = [...token];
+      for (let index = 0; index <= characters.length - 2; index += 1) {
+        tokens.push(characters.slice(index, index + 2).join(""));
+      }
+      for (let index = 0; index <= characters.length - 3; index += 1) {
+        tokens.push(characters.slice(index, index + 3).join(""));
+      }
+    }
+  }
+  return tokens;
+}
+
+function containsCjk(value) {
+  return /[\p{Script=Han}\p{Script=Hangul}\p{Script=Hiragana}\p{Script=Katakana}]/u.test(value);
+}