@ninjaxtools/slopdex 0.18.0 → 0.20.0

package/binary.js

@@ -0,0 +1,124 @@
+const { Package } = require("./binary-install");
+const os = require("os");
+const libc = require("detect-libc");
+
+const error = (msg) => {
+  console.error(msg);
+  process.exit(1);
+};
+
+const {
+  name,
+  artifactDownloadUrls,
+  supportedPlatforms,
+  glibcMinimum,
+} = require("./package.json");
+
+// FIXME: implement NPM installer handling of fallback download URLs
+const artifactDownloadUrl = artifactDownloadUrls[0];
+const builderGlibcMajorVersion = glibcMinimum.major;
+const builderGlibcMinorVersion = glibcMinimum.series;
+
+const getPlatform = () => {
+  const rawOsType = os.type();
+  const rawArchitecture = os.arch();
+
+  // We want to use rust-style target triples as the canonical key
+  // for a platform, so translate the "os" library's concepts into rust ones
+  let osType = "";
+  switch (rawOsType) {
+    case "Windows_NT":
+      osType = "pc-windows-msvc";
+      break;
+    case "Darwin":
+      osType = "apple-darwin";
+      break;
+    case "Linux":
+      osType = "unknown-linux-gnu";
+      break;
+  }
+
+  let arch = "";
+  switch (rawArchitecture) {
+    case "x64":
+      arch = "x86_64";
+      break;
+    case "arm64":
+      arch = "aarch64";
+      break;
+  }
+
+  if (rawOsType === "Linux") {
+    if (libc.familySync() == "musl") {
+      osType = "unknown-linux-musl-dynamic";
+    } else if (libc.isNonGlibcLinuxSync()) {
+      console.warn(
+        "Your libc is neither glibc nor musl; trying static musl binary instead",
+      );
+      osType = "unknown-linux-musl-static";
+    } else {
+      let libcVersion = libc.versionSync();
+      let splitLibcVersion = libcVersion.split(".");
+      let libcMajorVersion = splitLibcVersion[0];
+      let libcMinorVersion = splitLibcVersion[1];
+      if (
+        libcMajorVersion != builderGlibcMajorVersion ||
+        libcMinorVersion < builderGlibcMinorVersion
+      ) {
+        // We can't run the glibc binaries, but we can run the static musl ones
+        // if they exist
+        console.warn(
+          "Your glibc isn't compatible; trying static musl binary instead",
+        );
+        osType = "unknown-linux-musl-static";
+      }
+    }
+  }
+
+  // Assume the above succeeded and build a target triple to look things up with.
+  // If any of it failed, this lookup will fail and we'll handle it like normal.
+  let targetTriple = `${arch}-${osType}`;
+  let platform = supportedPlatforms[targetTriple];
+
+  if (!platform) {
+    error(
+      `Platform with type "${rawOsType}" and architecture "${rawArchitecture}" is not supported by ${name}.\nYour system must be one of the following:\n\n${Object.keys(
+        supportedPlatforms,
+      ).join(",")}`,
+    );
+  }
+
+  return platform;
+};
+
+const getPackage = () => {
+  const platform = getPlatform();
+  const url = `${artifactDownloadUrl}/${platform.artifactName}`;
+  let filename = platform.artifactName;
+  let ext = platform.zipExt;
+  let binary = new Package(platform, name, url, filename, ext, platform.bins);
+
+  return binary;
+};
+
+const install = (suppressLogs) => {
+  if (!artifactDownloadUrl || artifactDownloadUrl.length === 0) {
+    console.warn("in demo mode, not installing binaries");
+    return;
+  }
+  const pkg = getPackage();
+
+  return pkg.install(suppressLogs);
+};
+
+const run = (binaryName) => {
+  const pkg = getPackage();
+
+  pkg.run(binaryName);
+};
+
+module.exports = {
+  install,
+  run,
+  getPackage,
+};

package/install.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+
+const { install } = require("./binary");
+install(false);

package/npm-shrinkwrap.json

@@ -0,0 +1,52 @@
+{
+  "lockfileVersion": 3,
+  "name": "@ninjaxtools/slopdex",
+  "packages": {
+    "": {
+      "bin": {
+        "slopdex": "run-slopdex.js"
+      },
+      "dependencies": {
+        "detect-libc": "^2.1.2"
+      },
+      "devDependencies": {
+        "prettier": "^3.8.3"
+      },
+      "engines": {
+        "node": ">=14.14",
+        "npm": ">=6"
+      },
+      "hasInstallScript": true,
+      "license": "MIT",
+      "name": "@ninjaxtools/slopdex",
+      "version": "0.20.0"
+    },
+    "node_modules/detect-libc": {
+      "engines": {
+        "node": ">=8"
+      },
+      "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==",
+      "license": "Apache-2.0",
+      "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz",
+      "version": "2.1.2"
+    },
+    "node_modules/prettier": {
+      "bin": {
+        "prettier": "bin/prettier.cjs"
+      },
+      "dev": true,
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "url": "https://github.com/prettier/prettier?sponsor=1"
+      },
+      "integrity": "sha512-7igPTM53cGHMW8xWuVTydi2KO233VFiTNyF5hLJqpilHfmn8C8gPf+PS7dUT64YcXFbiMGZxS9pCSxL/Dxm/Jw==",
+      "license": "MIT",
+      "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.3.tgz",
+      "version": "3.8.3"
+    }
+  },
+  "requires": true,
+  "version": "0.20.0"
+}

package/package.json

@@ -1,72 +1,88 @@
 {
-  "name": "@ninjaxtools/slopdex",
-  "version": "0.18.0",
-  "description": "Tree-sitter callable-level semantic indexing, search, and duplicate discovery for Python, JavaScript, TypeScript, Rust, Go, Java, and C",
-  "license": "MIT",
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/ninjaxtools/slopdex.git"
+  "artifactDownloadUrls": [
+    "https://github.com/ninjaxtools/slopdex/releases/download/v0.20.0"
+  ],
+  "bin": {
+    "slopdex": "run-slopdex.js"
   },
-  "type": "module",
-  "main": "dist/index.js",
-  "types": "dist/index.d.ts",
-  "exports": {
-    ".": {
-      "types": "./dist/index.d.ts",
-      "import": "./dist/index.js"
-    }
+  "dependencies": {
+    "detect-libc": "^2.1.2"
   },
-  "bin": {
-    "slopdex": "dist/cli.js"
+  "description": "Semantic code and Markdown search with SQLite and USearch",
+  "devDependencies": {
+    "prettier": "^3.8.3"
   },
-  "files": [
-    ".agents/skills/slopdex/SKILL.md",
-    "dist",
-    "docs/implementation.md",
-    "docs/reference.md",
-    "README.md",
-    "scripts/install-opencode-skill.mjs"
-  ],
   "engines": {
-    "node": ">=24.0.0"
+    "node": ">=14.14",
+    "npm": ">=6"
+  },
+  "glibcMinimum": {
+    "major": 2,
+    "series": 39
   },
+  "license": "MIT",
+  "name": "@ninjaxtools/slopdex",
+  "preferUnplugged": true,
+  "repository": "https://github.com/ninjaxtools/slopdex",
   "scripts": {
-    "build": "tsup",
-    "dev": "tsx src/cli.ts",
-    "typecheck": "tsc --noEmit",
-    "test": "vitest",
-    "test:run": "vitest run",
-    "smoke": "node scripts/smoke.mjs",
-    "install:skill:opencode": "node scripts/install-opencode-skill.mjs",
-    "install:local": "npm run build && npm install -g .",
-    "publish:minor": "npm version minor && npm publish --access public",
-    "prepack": "npm run build && npm run smoke",
-    "check": "npm run typecheck && npm run test:run && npm run build && npm run smoke"
+    "fmt": "prettier --write **/*.js",
+    "fmt:check": "prettier --check **/*.js",
+    "postinstall": "node ./install.js"
   },
-  "dependencies": {
-    "@ai-sdk/anthropic": "^4.0.53",
-    "@ai-sdk/google": "^4.0.69",
-    "@ai-sdk/openai": "^4.0.66",
-    "@ai-sdk/openai-compatible": "^3.0.48",
-    "ai": "^7.0.99",
-    "ignore": "^7.0.9",
-    "install": "^0.13.0",
-    "js-tiktoken": "^1.0.21",
-    "sqlite-vec": "^0.1.9",
-    "tree-sitter": "^0.21.1",
-    "tree-sitter-c": "0.21.4",
-    "tree-sitter-go": "0.23.4",
-    "tree-sitter-java": "0.23.5",
-    "tree-sitter-javascript": "^0.23.1",
-    "tree-sitter-python": "0.23.2",
-    "tree-sitter-rust": "0.21.0",
-    "tree-sitter-typescript": "^0.23.2"
+  "supportedPlatforms": {
+    "aarch64-apple-darwin": {
+      "artifactName": "slopdex-aarch64-apple-darwin.tar.xz",
+      "bins": {
+        "slopdex": "slopdex"
+      },
+      "zipExt": ".tar.xz"
+    },
+    "aarch64-pc-windows-msvc": {
+      "artifactName": "slopdex-x86_64-pc-windows-msvc.zip",
+      "bins": {
+        "slopdex": "slopdex.exe"
+      },
+      "zipExt": ".zip"
+    },
+    "aarch64-unknown-linux-gnu": {
+      "artifactName": "slopdex-aarch64-unknown-linux-gnu.tar.xz",
+      "bins": {
+        "slopdex": "slopdex"
+      },
+      "zipExt": ".tar.xz"
+    },
+    "x86_64-apple-darwin": {
+      "artifactName": "slopdex-x86_64-apple-darwin.tar.xz",
+      "bins": {
+        "slopdex": "slopdex"
+      },
+      "zipExt": ".tar.xz"
+    },
+    "x86_64-pc-windows-gnu": {
+      "artifactName": "slopdex-x86_64-pc-windows-msvc.zip",
+      "bins": {
+        "slopdex": "slopdex.exe"
+      },
+      "zipExt": ".zip"
+    },
+    "x86_64-pc-windows-msvc": {
+      "artifactName": "slopdex-x86_64-pc-windows-msvc.zip",
+      "bins": {
+        "slopdex": "slopdex.exe"
+      },
+      "zipExt": ".zip"
+    },
+    "x86_64-unknown-linux-gnu": {
+      "artifactName": "slopdex-x86_64-unknown-linux-gnu.tar.xz",
+      "bins": {
+        "slopdex": "slopdex"
+      },
+      "zipExt": ".tar.xz"
+    }
   },
-  "devDependencies": {
-    "@types/node": "^24.0.0",
-    "tsup": "^8.5.0",
-    "tsx": "^4.20.0",
-    "typescript": "^5.9.0",
-    "vitest": "^3.2.0"
+  "version": "0.20.0",
+  "volta": {
+    "node": "18.14.1",
+    "npm": "9.5.0"
   }
-}
+}

package/run-slopdex.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+
+const { run } = require("./binary");
+run("slopdex");

package/.agents/skills/slopdex/SKILL.md

@@ -1,241 +0,0 @@
----
-name: slopdex
-description: Semantic code search, duplicate-function candidates, and physical-distance re-ranking
----
-
-# Slopdex operator guide for agents
-
-Slopdex does semantic code search, finds similar-code candidates, and identifies related functions stored far apart.
-
-### Find code by meaning
-
-```bash
-slopdex search "validate an authenticated session" --format summary --limit 10
-slopdex search "persist user data" -e 'save|persist' --format summary --limit 5
-```
-
-Describe behavior rather than guessing a symbol name. `-e` is a regex that restricts which symbols (functions) are searched.
-
-Hosted reranking is optional and persists in repository config:
-
-```bash
-slopdex config reranker cohere
-# Or: slopdex config reranker jina
-# Or use an LLM: slopdex config reranker openai
-```
-
-Set `COHERE_API_KEY`, `JINA_API_KEY`, or `OPENAI_API_KEY` respectively. The OpenAI LLM reranker defaults to `gpt-5.6-luna`, high reasoning, and the top 10 embedding candidates; configure the pool with `slopdex config reranker openai --reranker-candidates 20`. Disable reranking with `slopdex config reranker disable`. Reranking applies to `search` and `search-description`, not cross-search. It preserves embedding `similarity`, adds `rerankScore`, and orders a wider candidate set by that score.
-
-### Search function purpose
-
-Code purpose-description generation needs to be enabled once:
-
-```bash
-slopdex descriptions enable # only needed once
-```
-
-Then purpose descriptions can be searched:
-
-```bash
-slopdex search-description "keep the repository index synchronized" --format summary --limit 10
-```
-
-Enabling needs `OPENAI_API_KEY` by default. Select OpenCode Zen or Go with
-`--description-provider opencode` or `--description-provider opencode-go` and set `OPENCODE_API_KEY`
-or sign in with `opencode auth login`, which stores the key in `~/.local/share/opencode/auth.json`.
-
-To select another model:
-
-```bash
-slopdex descriptions enable --description-model <model-id>
-```
-
-List and persist a published OpenCode model without creating an index:
-
-```bash
-slopdex models opencode-go
-slopdex config model opencode-go/gpt-5.6-luna
-slopdex config descriptions enable
-```
-
-The next index-using command applies the configured description state. A bare model ID passed to
-`config model` resolves automatically only when it belongs to one of Zen or Go; qualify shared IDs.
-
-### Find duplicate candidates
-
-```bash
-slopdex cross-search --cross-file-only --min-lines 4 --threshold 0.9 --matches 5 --limit 5
-```
-
-The default output is connected clusters. This excludes same-file matches and short functions, keeps 5 matches per source, and emits at most 5 clusters. For source-by-source matches, add `--format summary`. `--limit` caps emitted clusters/sources (unlimited by default); `--matches` caps matches per source (default 5).
-
-Broaden discovery through adjacent score bands when needed:
-
-```bash
-slopdex cross-search --cross-file-only --min-lines 4 --threshold 0.85-0.9 --matches 5
-slopdex cross-search --cross-file-only --min-lines 4 --threshold 0.8-0.85 --matches 5
-```
-
-Ranges include the lower bound and exclude the upper bound. Use `--min-lines 1` when one-line wrappers are relevant.
-
-### Review changes or a module
-
-```bash
-slopdex cross-search --uncommitted --cross-file-only --min-lines 4 --threshold 0.9
-slopdex cross-search --changed-since origin/main --format summary --threshold 0.9
-slopdex cross-search --source-path src/services -e '^UserService\.' --format summary --threshold 0.9
-```
-
-These restrict sources while searching the full eligible index. All supplied restrictions intersect:
-
-```bash
-slopdex cross-search --source-path src -e 'validate' \
-  --changed-since origin/main --uncommitted \
-  --cross-file-only --min-lines 4 --threshold 0.9
-```
-
-Here a source must have changed since the commit and belong to an uncommitted file, within the selected path/name scope. `--regex` is an alias for `-e/--regexp`.
-
-### Review physical cohesion
-
-```bash
-slopdex cross-search --cohesion --threshold 0.8 --matches 20 --format summary
-slopdex cross-search --cohesion --source-path src/services --threshold 0.8 --format summary
-```
-
-`--cohesion` keeps cross-search's semantic matches and orders each source's matches from greatest to least physical path distance. Similarity breaks distance ties. Summary output includes the distance; JSONL matches include `physicalDistance`.
-
-### Compare repositories
-
-```bash
-slopdex cross-search \
-  --target-root /path/to/other/repo \
-  --target-index /path/to/other/repo/.slopdex/index.sqlite \
-  --threshold 0.9 --format summary
-```
-
-Both target options are required. Both indexes refresh and must have identical embedding profiles. The target refresh uses the source command's embedding provider and the target's file-selection/description configuration. Use `--target-config <path>` for a custom target config.
-
-### Inspect or maintain the index
-
-```bash
-slopdex status
-slopdex index-errors --format summary
-slopdex update-git
-slopdex update-files src/service.ts src/model.ts
-slopdex reindex-files
-slopdex reindex-files --callables
-slopdex delete-files src/removed.ts
-slopdex --version
-```
-
-- `status` refreshes, then reports coverage, checkpoint, profiles, and error counts; use when metadata is requested.
-- `index-errors` reads saved failures without refreshing or needing API credentials.
-- `update-git` explicitly refreshes HEAD and working-tree changes.
-- `update-files` reparses specified working-tree files after automatic refresh, even when their contents are unchanged.
-- `reindex-files` regenerates stale file descriptions and embeddings. Add `--callables` to also replace callable descriptions in those files.
-- `delete-files` removes index entries after automatic refresh, not source files. Eligible files can return on later refresh.
-- `--version` prints the built package version. `--help` describes available commands/options.
-
-## Command-line reference
-
-Usage: `slopdex <command> [arguments] [options]`. Quote queries and regexes. Boolean flags default to off. Use options only with their applicable commands.
-
-### General settings
-
-| Argument | Meaning / default |
-| --- | --- |
-| `--root <path>` | Repository root; current directory by default. |
-| `--config <path>` | Config; `<root>/.slopdex/config.json`. |
-| `--index <path>` | Index; `<root>/.slopdex/index.sqlite`. Overrides config `indexPath`. |
-| `--provider <openai\|jina>` | Embedding provider; `openai`. |
-| `--model <name>` | Embedding model; OpenAI `text-embedding-3-large`, Jina `jina-embeddings-v4`. |
-| `--dimensions <number>` | Positive dimensions supported by the model; OpenAI `3072`, Jina `1024`. |
-| `--description-provider <openai\|opencode\|opencode-go>` | Description provider; OpenAI by default. OpenCode values use `OPENCODE_API_KEY` or `~/.local/share/opencode/auth.json`. |
-| `--description-model <name>` | Description model; `gpt-5.6-luna` for OpenAI and `muse-spark-1.3-contributor` for Zen/Go. |
-| `--reranker-candidates <number>` | With `config reranker openai`, embedding-ranked functions sent to the LLM; range `1`-`100`, default `10`. |
-| `--ignore-errors` | Silence saved-diagnostic warnings without deleting records. |
-| `--verbose` | Report every external model request on stderr instead of once per call kind/provider/model. Config `"verbose": true` has the same effect. |
-| `-h`, `--help` | Usage; no refresh. |
-| `--version` | Package version; exits without refresh or saved-diagnostic warnings. |
-
-Explicit relative config/index paths resolve from the current directory. Source paths and explicit file arguments resolve within `--root`. Prefer absolute paths when operating across repositories. CLI settings override config.
-
-### Query and analysis options
-
-| Argument | Applies to / behavior |
-| --- | --- |
-| `--limit <number>` | Positive integer output limit; unlimited unless passed. Query matches, or cross-search clusters (`clusters`) / matched sources (`summary`/JSONL). Threshold filters results. |
-| `--matches <number>` | Cross-search only: matches kept per source function; default `5`. |
-| `--threshold <number\|min-max>` | Both query searches and cross-search. Inclusive minimum or half-open range; default `0.3`. |
-| `--format <json\|summary\|clusters>` | Both query searches, cross-search, and index-errors. Cohesion-ranked cross-search supports summary or JSONL, not clusters. |
-| `-e <regex>`, `--regexp <regex>`, `--regex <regex>` | Equivalent case-sensitive JavaScript regex options on qualified names. Query searches filter results before limiting; cross-search filters sources only. |
-| `--min-lines <number>` | Cross-search: positive source/candidate length minimum, default `2`. |
-| `--source-path <path>` | Cross-search: source file or recursive directory within the root. |
-| `--changed-since <commit>` | Cross-search: added, modified, or moved functions since an ancestor of the indexed Git checkpoint, including working-tree changes. Requires Git. |
-| `--uncommitted` | Cross-search: functions indexed from working-tree files; in Git these are staged, unstaged, or untracked changes. Without Git this selects all working-tree functions. |
-| `--cross-file-only` | Cross-search: exclude same-physical-file matches. |
-| `--include-symmetric-duplicates` | Cross-search: allow both directions of same-index matches; otherwise each unordered pair appears once. |
-| `--cohesion` | Cross-search: add physical distance and order each source's matches from farthest to nearest. Defaults to summary output. |
-| `--target-root <path>` | Cross-search: second repository; requires `--target-index`. |
-| `--target-index <path>` | Cross-search: second index file; requires `--target-root`. |
-| `--target-config <path>` | Cross-search: target config, default `<target-root>/.slopdex/config.json`; requires both target options. |
-
-### Refresh and recovery options
-
-| Argument | Behavior |
-| --- | --- |
-| `--target <ref>` | `update-git` snapshot, default `HEAD`. Non-HEAD targets are committed-only; later commands normally return to HEAD. |
-| `--rebuild-on-divergence` | Permit reconciliation after non-descendant history changes, such as a rebase/branch switch. |
-| `--force-reindex` | Recreate an incompatible index. Compatible indexes still use normal refresh; this is not an unconditional reparse flag. |
-| `--no-reindex` | With Git, reconcile the committed snapshot but omit working-tree overlays. Without Git, reuse a non-empty index; missing/empty indexes still populate. Not an offline mode. |
-| `--callables` | With `reindex-files`, continue after the file description and regenerate every callable description in each stale file. |
-
-Use `--no-reindex` when the task calls for committed-only results or reuse of an existing non-Git index, rather than silently weakening freshness.
-
-## Interpret and report results
-
-### Output formats
-
-| Command | Default | Alternatives |
-| --- | --- | --- |
-| `search`, `search-description` | `summary` | JSON array; purpose search includes generated description text |
-| `cross-search` | `clusters` | `summary`, or `json` for JSONL with one row per matched source |
-| `index-errors` | `summary` | JSON array |
-| `status`, update commands, `descriptions` | JSON object | — |
-
-Prefer summary output for compact source review, clusters for duplicate families, and JSON/JSONL for structured processing. Stdout carries results; stderr carries notices and warnings. External vector, description, and reranking requests identify their provider and model once per combination, or for every request with `--verbose`. Cross-search omits sources without emitted matches. Empty output means no findings under the chosen coverage/filters, not proof that no similar code exists.
-
-With reranking enabled, query summaries display both reranker relevance and embedding similarity. Similarity thresholds filter candidates before reranking; limits apply to the reranked output. LLM candidate documents include descriptions when available and function metadata/source code.
-
-### Similarity and clusters
-
-```text
-Cluster 1 (3 functions, similarity 0.9124-0.9568)
-  src/auth/session.ts:18:1 :: validateSession
-  src/http/middleware.ts:42:1 :: authenticate
-  src/users/user-service.ts:27:3 :: UserService.authenticate
-```
-
-- Similarity is a model-dependent resemblance score, not a duplication probability.
-- The range describes observed links. Members can be connected transitively; not all pairs necessarily match.
-- Cluster numbers reflect ordering by member count and name, not severity.
-- Inspect listed locations and callers. Tests, facades, adapters, and intentional layers can resemble each other without being redundant.
-
-When reporting candidates, identify paths/symbols, summarize the shared behavior you verified, and explain whether consolidation is appropriate. Do not infer equivalence from the score alone.
-
-### Purpose-aware scoring
-
-When descriptions are complete, `search` and cross-search average code, callable-description, and file-description similarity with equal one-third weights. `search-description` averages callable and file descriptions. Cross-repository analysis needs completeness on both sides; otherwise all scores are code-only. Stale file descriptions remain in scoring until `reindex-files` refreshes them. Description-generator models may differ even though embedding profiles must match.
-
-Thresholds and limits apply to the selected score. Text labels combined scoring; JSON includes component scores and mode/weights in `scoring`. Compare runs only with matching scoring mode, weights, embedding and description-generator profiles, threshold, and source/candidate filters.
-
-### Cohesion
-
-```text
-src/auth/session.ts :: validateSession
-  0.9400  packages/http/middleware.ts :: authenticate  [distance 4]
-  0.9300  src/auth/token.ts :: validateToken  [distance 1]
-```
-
-Physical distance is `0` within one file, `1` between files in one folder, and `1` plus directory-tree hops across folders. The option only reorders the selected semantic matches; it does not change similarity or prove that distant code should be moved. Review architectural layers, tests, adapters, and other intentional separation before recommending consolidation.