mikoshi 0.1.0

package/README.md

@@ -0,0 +1,41 @@
+```bash
+npm install -g mikoshi
+```
+
+✨ Mikoshi — private local code search + MCP
+
+## Install
+```bash
+npm install -g mikoshi
+```
+
+Use
+```bash
+mikoshi index ~/project
+mikoshi search ~/project "query"
+```
+
+MCP
+```bash
+mikoshi-mcp
+```
+
+Codex MCP config (config.toml)
+```toml
+[mcp_servers.mikoshi]
+command = "mikoshi-mcp"
+args = []
+enabled = true
+
+[projects."~/project"]
+trust_level = "trusted"
+```
+
+Developer
+```bash
+npm install
+npm test
+mikoshi doctor
+```
+
+Note: First run may download the local embedding model once.

package/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "mikoshi",
+  "version": "0.1.0",
+  "description": "Private local code search + MCP",
+  "type": "module",
+  "bin": {
+    "mikoshi": "src/mikoshi/cli.js",
+    "mikoshi-mcp": "src/mikoshi/mcp_server/server.js"
+  },
+  "files": [
+    "src/mikoshi/**",
+    "README.md"
+  ],
+  "scripts": {
+    "test": "node --test",
+    "lint": "node --check ./src/mikoshi/cli.js"
+  },
+  "dependencies": {},
+  "optionalDependencies": {
+    "@xenova/transformers": "^2.17.2",
+    "onnxruntime-node": "^1.20.0"
+  }
+}

package/src/mikoshi/auth.js

@@ -0,0 +1,199 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import crypto from "node:crypto";
+import readline from "node:readline";
+import { DEFAULT_FEATURES, DEFAULT_PLAN } from "./entitlements.js";
+
+export const DEFAULT_API_BASE_URL = "https://neet.gg";
+
+const AUTH_FILENAME = "auth.json";
+const CONFIG_FILENAME = "config.json";
+
+export class AuthError extends Error {}
+
+function indexRoot() {
+  const root = process.env.MIKOSHI_INDEX_ROOT || "~/.mikoshi";
+  return path.resolve(root.replace(/^~\//, `${os.homedir()}/`));
+}
+
+export function authPath() {
+  return path.join(indexRoot(), AUTH_FILENAME);
+}
+
+export function configPath() {
+  return path.join(indexRoot(), CONFIG_FILENAME);
+}
+
+export function loadBrokerConfig() {
+  const cfgPath = configPath();
+  if (!fs.existsSync(cfgPath)) {
+    return { api_base_url: DEFAULT_API_BASE_URL };
+  }
+  try {
+    const data = JSON.parse(fs.readFileSync(cfgPath, "utf8"));
+    const apiBase = String(data.api_base_url || "").trim().replace(/\/+$/, "");
+    return { api_base_url: apiBase || DEFAULT_API_BASE_URL };
+  } catch {
+    return { api_base_url: DEFAULT_API_BASE_URL };
+  }
+}
+
+export function saveBrokerConfig(apiBaseUrl) {
+  const cleanUrl = String(apiBaseUrl || "").trim().replace(/\/+$/, "") || DEFAULT_API_BASE_URL;
+  const cfgPath = configPath();
+  fs.mkdirSync(path.dirname(cfgPath), { recursive: true });
+  const payload = JSON.stringify({ api_base_url: cleanUrl }, null, 2);
+  const tmp = cfgPath + ".tmp";
+  fs.writeFileSync(tmp, payload, { mode: 0o600 });
+  fs.renameSync(tmp, cfgPath);
+  fs.chmodSync(cfgPath, 0o600);
+}
+
+export function loadAuthState() {
+  const p = authPath();
+  if (!fs.existsSync(p)) return null;
+  try {
+    const data = JSON.parse(fs.readFileSync(p, "utf8"));
+    return {
+      access_token: String(data.access_token || ""),
+      expires_at: String(data.expires_at || ""),
+      plan: String(data.plan || DEFAULT_PLAN),
+      features: Array.isArray(data.features) ? data.features.map(String) : DEFAULT_FEATURES,
+    };
+  } catch {
+    return null;
+  }
+}
+
+export function saveAuthState(state) {
+  const p = authPath();
+  fs.mkdirSync(path.dirname(p), { recursive: true });
+  const payload = JSON.stringify(
+    {
+      access_token: state.access_token,
+      expires_at: state.expires_at,
+      plan: state.plan || DEFAULT_PLAN,
+      features: state.features || DEFAULT_FEATURES,
+    },
+    null,
+    2
+  );
+  const tmp = p + ".tmp";
+  fs.writeFileSync(tmp, payload, { mode: 0o600 });
+  fs.renameSync(tmp, p);
+  fs.chmodSync(p, 0o600);
+}
+
+export function clearAuthState() {
+  const p = authPath();
+  if (fs.existsSync(p)) fs.unlinkSync(p);
+}
+
+export function isExpired(state) {
+  if (!state?.expires_at) return true;
+  const value = state.expires_at.endsWith("Z")
+    ? state.expires_at.slice(0, -1) + "+00:00"
+    : state.expires_at;
+  const parsed = Date.parse(value);
+  if (Number.isNaN(parsed)) return true;
+  return Date.now() >= parsed;
+}
+
+export function emailFromToken(token) {
+  try {
+    const parts = token.split(".");
+    if (parts.length < 2) return null;
+    const payload = parts[1] + "=".repeat((4 - (parts[1].length % 4)) % 4);
+    const decoded = JSON.parse(Buffer.from(payload, "base64url").toString("utf8"));
+    return decoded.email || decoded.user_email || decoded.preferred_username || null;
+  } catch {
+    return null;
+  }
+}
+
+function generateState() {
+  return crypto.randomBytes(32).toString("base64url");
+}
+
+function generateCodeVerifier() {
+  return crypto.randomBytes(64).toString("base64url");
+}
+
+function codeChallenge(verifier) {
+  return crypto.createHash("sha256").update(verifier).digest("base64url");
+}
+
+function loginUrl(apiBaseUrl, state, challenge) {
+  const params = new URLSearchParams({
+    response_type: "code",
+    client_id: "mikoshi",
+    code_challenge: challenge,
+    code_challenge_method: "S256",
+    state,
+  });
+  return `${apiBaseUrl.replace(/\/+$/, "")}/login?${params.toString()}`;
+}
+
+async function openUrl(url) {
+  const platform = process.platform;
+  const command = platform === "darwin" ? "open" : platform === "win32" ? "start" : "xdg-open";
+  const { spawn } = await import("node:child_process");
+  spawn(command, [url], { stdio: "ignore", detached: true }).unref();
+}
+
+async function promptLine(prompt) {
+  const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+  const answer = await new Promise((resolve) => rl.question(prompt, resolve));
+  rl.close();
+  return String(answer || "");
+}
+
+function parsePastePayload(text) {
+  let data;
+  try {
+    data = JSON.parse(text);
+  } catch (err) {
+    throw new AuthError("Invalid JSON response.");
+  }
+  if (!data || typeof data !== "object") throw new AuthError("Invalid JSON response.");
+  const code = String(data.code || "");
+  const state = String(data.state || "");
+  if (!code || !state) throw new AuthError("Missing code or state.");
+  return { code, state };
+}
+
+export async function login() {
+  const broker = loadBrokerConfig();
+  const state = generateState();
+  const verifier = generateCodeVerifier();
+  const challenge = codeChallenge(verifier);
+  const url = loginUrl(broker.api_base_url, state, challenge);
+
+  await openUrl(url);
+
+  const pasted = await promptLine("Paste the JSON response here: ");
+  const { code, state: returnedState } = parsePastePayload(pasted);
+  if (returnedState !== state) {
+    throw new AuthError("Invalid state.");
+  }
+
+  const response = await fetch(`${broker.api_base_url.replace(/\/+$/, "")}/cli/exchange`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ code, code_verifier: verifier, state }),
+  });
+  if (!response.ok) {
+    throw new AuthError("Login failed. Please try again.");
+  }
+  const payload = await response.json();
+  const accessToken = String(payload.access_token || "");
+  const expiresAt = String(payload.expires_at || "");
+  if (!accessToken || !expiresAt) throw new AuthError("Login failed. Please try again.");
+
+  const plan = String(payload.plan || DEFAULT_PLAN);
+  const features = Array.isArray(payload.features) ? payload.features.map(String) : DEFAULT_FEATURES;
+  saveAuthState({ access_token: accessToken, expires_at: expiresAt, plan, features });
+
+  return String(payload.email || emailFromToken(accessToken) || "unknown");
+}

package/src/mikoshi/chunking.js

@@ -0,0 +1,31 @@
+/**
+ * @typedef {{start_line:number,end_line:number,text:string}} ChunkSpan
+ */
+
+/**
+ * Deterministic line-based chunking with overlap.
+ * @param {string} text
+ * @param {number} maxLines
+ * @param {number} overlap
+ * @returns {ChunkSpan[]}
+ */
+export function chunkText(text, maxLines, overlap) {
+  const lines = text.split(/\r?\n/);
+  if (maxLines <= 0) return [];
+
+  const spans = [];
+  let start = 0;
+  while (start < lines.length) {
+    const end = Math.min(start + maxLines, lines.length);
+    const slice = lines.slice(start, end);
+    const chunkText = slice.join("\n");
+    spans.push({
+      start_line: start + 1,
+      end_line: end,
+      text: chunkText,
+    });
+    if (end === lines.length) break;
+    start = Math.max(0, end - overlap);
+  }
+  return spans;
+}

package/src/mikoshi/cli.js

@@ -0,0 +1,274 @@
+#!/usr/bin/env node
+import fs from "node:fs";
+import path from "node:path";
+import { configureExternalLibs, ConfigError, loadConfig } from "./config.js";
+import { indexRepo } from "./indexing/indexer.js";
+import { IndexStore } from "./indexing/index_store.js";
+import { searchRepo } from "./retrieval/hybrid.js";
+import {
+  AuthError,
+  DEFAULT_API_BASE_URL,
+  clearAuthState,
+  emailFromToken,
+  isExpired,
+  loadAuthState,
+  login,
+  saveBrokerConfig,
+} from "./auth.js";
+
+function formatIndexRoot(rootPath) {
+  try {
+    const home = path.resolve(process.env.HOME || "");
+    const resolved = path.resolve(rootPath);
+    if (resolved === home) return "~";
+    if (resolved.startsWith(home + path.sep)) {
+      return resolved.replace(home, "~");
+    }
+    return resolved;
+  } catch {
+    return rootPath;
+  }
+}
+
+async function cmdIndex(args) {
+  try {
+    const result = await indexRepo(args.path);
+    console.log(JSON.stringify({
+      repo_id: result.repo_id,
+      chunks_indexed: result.chunks_indexed,
+      took_ms: result.took_ms,
+    }, null, 2));
+    return 0;
+  } catch (err) {
+    if (err instanceof ConfigError) {
+      console.error(`Config error: ${err.message}`);
+      return 2;
+    }
+    console.error(String(err.message || err));
+    return 1;
+  }
+}
+
+async function cmdSearch(args) {
+  try {
+    const results = await searchRepo(args.path, args.query, args.k);
+    for (const result of results) {
+      console.log(`${result.relpath}:${result.start_line}-${result.end_line} (${result.score.toFixed(3)})`);
+      console.log(result.snippet);
+      console.log();
+    }
+    return 0;
+  } catch (err) {
+    if (err instanceof ConfigError) {
+      console.error(`Config error: ${err.message}`);
+      return 2;
+    }
+    console.error(String(err.message || err));
+    return 1;
+  }
+}
+
+function cmdStatus(args) {
+  const config = loadConfig();
+  const repoRoot = path.resolve(args.path);
+  const store = new IndexStore(repoRoot, config.index_root);
+  const meta = store.loadMeta();
+  if (!meta) {
+    console.log(JSON.stringify({ indexed: false, chunks: 0, last_index_time: null, model: null }, null, 2));
+    return 0;
+  }
+  console.log(JSON.stringify({
+    indexed: true,
+    chunks: meta.chunks,
+    last_index_time: meta.updated_at,
+    model: meta.model,
+  }, null, 2));
+  return 0;
+}
+
+function cmdClear(args) {
+  const config = loadConfig();
+  const repoRoot = path.resolve(args.path);
+  const store = new IndexStore(repoRoot, config.index_root);
+  store.clear();
+  console.log(JSON.stringify({ ok: true }, null, 2));
+  return 0;
+}
+
+function cmdDoctor(args) {
+  let exitCode = 0;
+  const version = process.versions.node.split(".").slice(0, 2).join(".");
+  const major = Number.parseInt(process.versions.node.split(".")[0], 10);
+  if (major >= 20) {
+    console.log(`✅ Node: ${version}.x`);
+  } else {
+    console.log(`❌ Node: ${version}.x (requires 20+)`);
+    exitCode = 1;
+  }
+
+  let config;
+  try {
+    config = loadConfig();
+  } catch (err) {
+    console.log(`❌ Config: ${err.message}`);
+    return 1;
+  }
+
+  console.log(`✅ Mikoshi index root: ${formatIndexRoot(config.index_root)}`);
+  console.log(`✅ Embeddings: provider=${config.embeddings.provider} model=${config.embeddings.model}`);
+
+  if (config.embeddings.provider === "local") {
+    const cacheDir = path.join(config.index_root, "models");
+    const cached = fs.existsSync(cacheDir) && fs.readdirSync(cacheDir).length > 0;
+    const offline = ["1", "true", "yes", "on"].includes((process.env.MIKOSHI_OFFLINE || "").toLowerCase());
+    if (!cached && offline) {
+      console.log("❌ Model cached: no (offline)");
+      exitCode = 1;
+    } else {
+      console.log(`✅ Model cached: ${cached ? "yes" : "no"}`);
+    }
+  } else {
+    console.log("✅ Model cached: yes");
+  }
+
+  if (args.path) {
+    const repoRoot = path.resolve(args.path);
+    const store = new IndexStore(repoRoot, config.index_root);
+    const meta = store.loadMeta();
+    if (meta) {
+      console.log(`✅ Repo indexed: yes (chunks=${meta.chunks}, last_index_time=${meta.updated_at})`);
+    } else {
+      console.log("✅ Repo indexed: no");
+    }
+  }
+
+  return exitCode;
+}
+
+async function cmdLogin() {
+  const state = loadAuthState();
+  if (state && !isExpired(state)) {
+    console.log("⚠️ You are already logged in to Mikoshi.");
+    console.log("Re-authenticating will replace your current session.");
+    const answer = await promptLine(
+      "Do you want to continue with re-authentication? This will invalidate your existing session if successful. (y/N): "
+    );
+    if (!new Set(["y", "yes"]).has(answer.trim().toLowerCase())) return 0;
+  }
+  try {
+    console.log("🔐 Starting NEET authentication...");
+    console.log("🌐 Opening authentication page in your browser...");
+    const email = await login();
+    console.log(`✅ Logged in as ${email}`);
+    return 0;
+  } catch (err) {
+    if (err instanceof AuthError) {
+      console.error(err.message);
+      return 2;
+    }
+    console.error(String(err.message || err));
+    return 1;
+  }
+}
+
+function cmdLogout() {
+  clearAuthState();
+  console.log("✅ Logged out");
+  return 0;
+}
+
+function cmdWhoami() {
+  const state = loadAuthState();
+  if (!state || isExpired(state)) {
+    console.log("🔒 Not signed in");
+    return 1;
+  }
+  const email = emailFromToken(state.access_token) || "unknown";
+  const plan = state.plan ? state.plan[0].toUpperCase() + state.plan.slice(1) : "Free";
+  console.log(`✅ ${email} (${plan})`);
+  return 0;
+}
+
+async function cmdAuthConfigure() {
+  const answer = await promptLine(`API base URL [${DEFAULT_API_BASE_URL}]: `);
+  const apiBase = answer.trim() || DEFAULT_API_BASE_URL;
+  try {
+    saveBrokerConfig(apiBase);
+  } catch (err) {
+    console.error(err.message || err);
+    return 2;
+  }
+  console.log("✅ Auth configured");
+  return 0;
+}
+
+function parseArgs(argv) {
+  const args = { _: [] };
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === "--verbose") {
+      args.verbose = true;
+    } else {
+      args._.push(arg);
+    }
+  }
+  return args;
+}
+
+async function promptLine(prompt) {
+  const rl = (await import("node:readline")).createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+  const answer = await new Promise((resolve) => rl.question(prompt, resolve));
+  rl.close();
+  return String(answer || "");
+}
+
+async function main() {
+  const parsed = parseArgs(process.argv.slice(2));
+  if (parsed.verbose) {
+    process.env.MIKOSHI_QUIET_LIBS = "0";
+  }
+
+  const config = loadConfig();
+  configureExternalLibs(config.quiet_external_libs);
+
+  const [command, ...rest] = parsed._;
+  switch (command) {
+    case "index":
+      return process.exit(await cmdIndex({ path: rest[0] }));
+    case "search":
+      return process.exit(await cmdSearch({ path: rest[0], query: rest[1], k: Number(rest[2] || 8) }));
+    case "status":
+      return process.exit(cmdStatus({ path: rest[0] }));
+    case "clear":
+      return process.exit(cmdClear({ path: rest[0] }));
+    case "doctor":
+      return process.exit(cmdDoctor({ path: rest[0] }));
+    case "login":
+      return process.exit(await cmdLogin());
+    case "logout":
+      return process.exit(cmdLogout());
+    case "whoami":
+      return process.exit(cmdWhoami());
+    case "auth":
+      if (rest[0] === "configure") return process.exit(await cmdAuthConfigure());
+      break;
+    default:
+      break;
+  }
+
+  console.log("Mikoshi CLI");
+  console.log("Usage:");
+  console.log("  mikoshi index <path>");
+  console.log("  mikoshi search <path> <query> [k]");
+  console.log("  mikoshi status <path>");
+  console.log("  mikoshi clear <path>");
+  console.log("  mikoshi doctor [path]");
+  console.log("  mikoshi login | logout | whoami");
+  console.log("  mikoshi auth configure");
+  process.exit(1);
+}
+
+main();

package/src/mikoshi/config.js

@@ -0,0 +1,58 @@
+import os from "node:os";
+import path from "node:path";
+
+export class ConfigError extends Error {}
+
+const DEFAULT_MODEL = "sentence-transformers/all-MiniLM-L6-v2";
+
+export function configureExternalLibs(quiet) {
+  if (!quiet) return;
+  process.env.HF_HUB_DISABLE_PROGRESS_BARS = "1";
+  process.env.TRANSFORMERS_VERBOSITY = "error";
+  process.env.TOKENIZERS_PARALLELISM = "false";
+  process.env.PYTHONWARNINGS = "ignore";
+}
+
+function parseIntEnv(name, fallback) {
+  const raw = process.env[name];
+  if (!raw) return fallback;
+  const value = Number.parseInt(raw, 10);
+  return Number.isFinite(value) ? value : fallback;
+}
+
+export function loadConfig() {
+  const indexRoot = process.env.MIKOSHI_INDEX_ROOT
+    ? path.resolve(process.env.MIKOSHI_INDEX_ROOT.replace(/^~\//, `${os.homedir()}/`))
+    : path.join(os.homedir(), ".mikoshi");
+
+  const provider = (process.env.MIKOSHI_EMBEDDINGS_PROVIDER || "local").toLowerCase();
+  const model = process.env.MIKOSHI_EMBED_MODEL || DEFAULT_MODEL;
+  const openaiKey = process.env.MIKOSHI_OPENAI_API_KEY || process.env.OPENAI_API_KEY || "";
+  const openaiBaseUrl =
+    process.env.MIKOSHI_OPENAI_BASE_URL || process.env.OPENAI_BASE_URL || "https://api.openai.com";
+  const openaiModel = process.env.MIKOSHI_OPENAI_EMBED_MODEL || "text-embedding-3-small";
+
+  if (provider !== "local" && provider !== "openai") {
+    throw new ConfigError(`Unknown embeddings provider: ${provider}`);
+  }
+  if (provider === "openai" && !openaiKey) {
+    throw new ConfigError(
+      "OpenAI embeddings provider selected but API key missing. Set MIKOSHI_OPENAI_API_KEY or OPENAI_API_KEY."
+    );
+  }
+
+  return {
+    index_root: indexRoot,
+    quiet_external_libs: process.env.MIKOSHI_QUIET_LIBS !== "0",
+    max_bytes: parseIntEnv("MIKOSHI_MAX_BYTES", 2_000_000),
+    chunk_lines: parseIntEnv("MIKOSHI_CHUNK_LINES", 200),
+    chunk_overlap: parseIntEnv("MIKOSHI_CHUNK_OVERLAP", 40),
+    embeddings: {
+      provider,
+      model,
+      openai_key: openaiKey,
+      openai_base_url: openaiBaseUrl,
+      openai_model: openaiModel,
+    },
+  };
+}

package/src/mikoshi/entitlements.js

@@ -0,0 +1,6 @@
+export const DEFAULT_PLAN = "free";
+export const DEFAULT_FEATURES = ["local_index", "mcp"];
+
+export function hasFeature(state, feature) {
+  return state?.features?.includes(feature);
+}

package/src/mikoshi/hashing.js

@@ -0,0 +1,9 @@
+import { createHash } from "node:crypto";
+
+export function sha256Bytes(buffer) {
+  return createHash("sha256").update(buffer).digest("hex");
+}
+
+export function sha256Text(text) {
+  return createHash("sha256").update(text, "utf8").digest("hex");
+}