@uehaj/sys1grep 0.5.0-next.1 → 0.5.0

package/examples/app.js

@@ -0,0 +1,49 @@
+// A small demo service: an entry point calls through a few layers down to where things can go wrong.
+export async function main() {
+  await handleRequest();
+}
+
+async function handleRequest() {
+  try {
+    await processOrder();
+  } catch (err) {
+    throw err;
+  }
+}
+
+async function processOrder() {
+  await chargeCard();
+}
+
+async function chargeCard() {
+  if (!cardIsValid()) fatal('card declined');
+  throw new Error('card declined');
+}
+
+function cardIsValid() {
+  return false;
+}
+
+function fatal(message) {
+  console.error(message);
+  process.exit(1);
+}
+
+// reads the API key from the environment first, then a local settings file
+function loadApiKey() {
+  return process.env.SYS1GREP_API_KEY || readSettingsFile().apiKey;
+}
+
+function readSettingsFile() {
+  return { apiKey: '' };
+}
+
+function tryOptionalStep() {
+  try {
+    riskyStep();
+  } catch (err) {} // ignored: this step is optional and failing it is not fatal
+}
+
+function riskyStep() {
+  return true;
+}

package/examples/app.py

@@ -0,0 +1,54 @@
+"""A small demo CLI: the same kind of call chain as app.js, in Python."""
+import os
+import sys
+
+
+def main():
+    run_job()
+
+
+def run_job():
+    try:
+        execute()
+    except Exception:
+        raise
+
+
+def execute():
+    if not is_ready():
+        raise RuntimeError('not ready')
+    charge()
+
+
+def is_ready():
+    return False
+
+
+def charge():
+    if not confirm_charge():
+        sys.exit(1)
+    raise RuntimeError('charge failed')
+
+
+def confirm_charge():
+    return False
+
+
+def load_api_key():
+    # reads the API key from the environment first, then a local settings file
+    return os.environ.get('SYS1GREP_API_KEY') or read_settings_file()
+
+
+def read_settings_file():
+    return ''
+
+
+def try_optional_step():
+    try:
+        risky_step()
+    except Exception:
+        pass  # ignored: optional, not fatal
+
+
+def risky_step():
+    return True

package/examples/logs.txt

@@ -0,0 +1,19 @@
+2026-01-05 03:12:01 INFO  server started, listening on port 8080
+2026-01-05 03:12:04 INFO  connected to the primary database
+2026-01-05 03:14:40 WARN  slow query took 2800ms: SELECT * FROM orders WHERE status = 'pending'
+2026-01-05 03:20:11 ERROR connection reset by peer while calling the payment service
+2026-01-05 03:20:12 INFO  retrying the payment request (attempt 2 of 3)
+2026-01-05 03:20:18 ERROR timeout after 5000ms waiting for the payment service
+2026-01-05 03:20:19 INFO  retrying the payment request (attempt 3 of 3)
+2026-01-05 03:20:25 FATAL giving up after 3 attempts: the payment service is unreachable
+2026-01-05 04:02:00 INFO  nightly backup completed
+2026-01-05 04:05:30 WARN  disk usage at 92% on /var/log
+2026-01-05 04:10:44 ERROR DNS lookup failed for the billing API
+2026-01-05 04:10:46 INFO  DNS lookup succeeded on retry
+2026-01-05 05:00:00 ERROR health check failed: no response from the application server
+2026-01-05 05:00:05 ERROR health check failed again: no response from the application server
+2026-01-05 05:00:10 FATAL marking the application server as down after 2 failed health checks
+2026-01-05 05:30:00 INFO  the application server is back and answering health checks
+2026-01-05 06:00:00 WARN  an admin ran DROP TABLE sessions; on the staging database by mistake
+2026-01-05 06:05:00 INFO  restored the sessions table from last night's backup
+2026-01-05 07:00:00 INFO  it is a quiet morning, nothing else to report

package/examples/prose.txt

@@ -0,0 +1,10 @@
+I have to admit, the bug was my fault: I shipped the migration without testing it on a copy of production data first.
+Looking back, I should have reviewed the diff more carefully before merging; the mistake was mine, not the reviewer's.
+The weather today is mild, with a light breeze from the west and clear skies through the evening.
+Lunch today was a simple rice bowl with grilled vegetables from the place around the corner.
+Next Friday is a public holiday, so the office will be closed for the long weekend.
+会議は明日の10時からに変更になりました。資料は前日までに共有してください。
+今日の天気は晴れ、最高気温は28度の見込みです。
+料金プランの変更方法については、設定画面の「プラン」から行えます。
+振り返ると、あのバグは自分のミスでした。テストを十分にせずにリリースしてしまったのが原因です。
+新機能のリリースは来月に延期されましたが、理由は品質を優先したためです。

package/examples/tickets.txt

@@ -0,0 +1,9 @@
+Ticket #4821 (support, EN): I want my money back. The item arrived broken and nobody answered my emails.
+Ticket #4822 (support, EN): Could you change the delivery address for order #910? I moved last week.
+Ticket #4823 (support, EN): This is infuriating. Three days with no reply and my account is still locked.
+Ticket #4824 (support, EN): Thanks so much, the replacement part fixed it and the team was lovely.
+Ticket #4825 (support, EN): The app crashed once but it worked fine the second time, no need for a refund.
+チケット #4826（サポート, JA）: 注文した覚えのない請求が来ています。至急確認してください。本当に困っています。
+チケット #4827（サポート, JA）: 配送先の住所を変更したいです。来週には新しい住所に届けてほしいです。
+チケット #4828（サポート, JA）: 対応いただきありがとうございました。問題は解決しました。
+チケット #4829（サポート, JA）: サポートに何度連絡しても返事がなく、とても腹が立っています。

package/lib/sentences.mjs

@@ -0,0 +1,49 @@
+// Scripts written without spaces between words: joining their wrapped lines must not add a space between them,
+// and sys1grep's token estimate prices them per character (cjkBytesOf).
+export const CJK = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}\p{Script=Tibetan}ー、。]/u;
+
+// --unit=sentence-by-*: the unit is a sentence. Lines are joined as wrapped prose, except where a newline cannot be inside
+// a sentence: at a blank line, next to structure characters (JSON, code), or before a list item, heading, quote or
+// number. Each joined piece is then split by Intl.Segmenter (Unicode UAX #29 sentence boundaries).
+const SENTENCES = new Intl.Segmenter(undefined, { granularity: 'sentence' });
+const hardBreak = (prev, next) => !prev || !next || /[{}\[\]<>|;]$/.test(prev) || /^[{}\[\]<>|"\-*+#>\d]/.test(next)
+  || /[{}\[\];]$/.test(next); // a line ending like code is not a continuation of prose either
+// lines -> [{ text, spans }]. A span [unit, from, to] is where the sentence lies in the original units: the unit
+// is the 1-based line (or record) number from unitOf, from/to are character offsets inside it (baseOf shifts a
+// line's offset inside its record with --unit=zero). Output maps matching sentences back to these units.
+export function toSentences(lines, unitOf, baseOf, split = new Set()) { // split: breaks Jev judged to end an entry
+  const out = [];
+  let text = '', marks = []; // marks: { o: offset in text, unit, base: offset in the unit, len }
+  const flush = () => {
+    for (const seg of SENTENCES.segment(text)) {
+      const t = seg.segment.trim();
+      if (!t) continue;
+      const start = seg.index + seg.segment.length - seg.segment.trimStart().length, end = start + t.length;
+      const spans = [];
+      for (const m of marks) {
+        const a = Math.max(start, m.o), b = Math.min(end, m.o + m.len);
+        if (a < b) spans.push([m.unit, m.base + a - m.o, m.base + b - m.o]);
+      }
+      out.push({ text: t, spans });
+    }
+    text = ''; marks = [];
+  };
+  lines.forEach((line, i) => {
+    const prev = text.trimEnd(), next = line.trim();
+    const mark = { unit: unitOf(i), base: baseOf(i) + line.length - line.trimStart().length, len: next.length };
+    if (hardBreak(prev, next) || split.has(i)) { flush(); if (next) { text = next; marks.push({ ...mark, o: 0 }); } return; }
+    text = prev + (CJK.test(prev.at(-1)) && CJK.test(next[0]) ? '' : ' '); // no space for scripts without word spaces
+    marks.push({ ...mark, o: text.length });
+    text += next;
+  });
+  flush();
+  return out;
+}
+
+// Where the rules would join, a break still needs judging if a script without word spaces touches it and the line
+// does not end a sentence: in Japanese or Chinese, entries often end without 。, and joining leaves no trace of the
+// break. A line starting with closing punctuation continues the previous one (kinsoku), and so does a line
+// ending in 、.
+const CLOSING = /^[。、，．」』）】〕！？]/;
+export const needsJudging = (prev, next) => prev && next && !hardBreak(prev, next) && !/[。！？!?、，]$/.test(prev) && !CLOSING.test(next)
+  && (CJK.test(prev.at(-1)) || CJK.test(next[0]));

package/options.mjs

@@ -0,0 +1,101 @@
+// sys1grep.mjs's own node:util.parseArgs option table and default-options gate, pulled out so serve.mjs's
+// settings-panel opts validation runs the exact same parser instead of a hand-rolled regex/cluster check that kept
+// missing forms the parser itself already understands (#199: -nr, -rC1, --r, -h all slipped past earlier regexes
+// in turn; parseArgs splits a cluster, resolves a glued value and a long alias for a one-letter name correctly by
+// construction, so there is nothing left to keep in sync by hand).
+import { parseArgs } from 'node:util';
+
+export const OPTIONS = {
+  e: { type: 'string', multiple: true },
+  P: { type: 'string', multiple: true }, // a regex term (/RE/flags), OR'd like -e; -a/-v after it AND/NOT onto it
+  a: { type: 'string', multiple: true },
+  v: { type: 'string', multiple: true },
+  question: { type: 'string', multiple: true, short: 'Q' },
+  level: { type: 'string', default: 'normal' }, // strictness preset: loose / normal / strict
+  r: { type: 'boolean', default: false }, // recurse into directories
+  cached: { type: 'boolean', default: false }, // git sys1grep only: search the index instead of the working tree
+  untracked: { type: 'boolean', default: false }, // git sys1grep only: also search untracked files (.gitignore still applies)
+  l: { type: 'boolean', default: false }, // print only matching file names
+  'with-filename': { type: 'boolean', short: 'H' }, // prefix file names even for one file; --no-filename: never
+  t: { type: 'string' }, // positive threshold: match when p >= t (default from preset)
+  T: { type: 'string' }, // negative threshold: "not X" when p < T (default from preset)
+  chunk: { type: 'string', default: '30' }, // lines per request
+  c: { type: 'boolean', default: false }, // count of matching lines per file (grep -c)
+  quiet: { type: 'boolean', short: 'q', default: false }, // print nothing, exit status only (grep -q)
+  j: { type: 'string', default: '8' }, // concurrent requests
+  A: { type: 'string' }, // N lines of trailing context
+  B: { type: 'string' }, // N lines of leading context
+  C: { type: 'string' }, // N lines of context on both sides
+  n: { type: 'boolean', default: false }, // line numbers
+  z: { type: 'boolean', default: false }, // records are NUL-terminated, on input and output (grep -z); --unit=zero
+  unit: { type: 'string', default: 'line' }, // line / zero / sentence-by-jev / sentence-by-rule / function
+  o: { type: 'boolean', default: false }, // with --unit=sentence-by-*, print only the matching sentences (grep -o)
+  p: { type: 'boolean', default: false }, // print each meaning's probability
+  dedup: { type: 'string', default: 'never' }, // auto|always|never: judge one representative per template, reuse its answer
+  rank: { type: 'string' }, // jev|match: print the results best first, scored by Jev's answer on each or by its best match
+  // multi-step matching (#163): the expression before --step-to finds the start, the one after it the end;
+  // --step-reverse-to walks the edges backwards (callee to caller)
+  'step-to': { type: 'boolean', multiple: true },
+  'step-reverse-to': { type: 'boolean', multiple: true },
+  edges: { type: 'string' }, // FILE: the edges to walk, one per line; default: the calls between functions
+  reverse: { type: 'boolean' }, // removed (--step-reverse-to): parsed only to say so
+  hops: { type: 'string', default: '0..' }, // N, M..N or M..: the hops an end may be at
+  'dry-run': { type: 'boolean', default: false }, // print the files and requests, send nothing
+  verbose: { type: 'boolean', default: false }, // print the files and requests to stderr while searching
+  interactive: { type: 'boolean', short: 'i', default: false }, // show what --dry-run would send, search on a yes
+  yes: { type: 'boolean', short: 'y', default: false }, // skip the size/cost guard's question, answering yes (#58)
+  'max-columns': { type: 'string', short: 'M' }, // a unit past this many characters is skipped (rg's -M/--max-columns)
+  'max-filesize': { type: 'string' }, // a target past this size (K/M/G) is listed and confirmed (rg's name)
+  'max-cost': { type: 'string', default: '1' }, // ask before sending when the estimated price is over this many USD
+  // which files -r finds and git sys1grep lists; with -r a file named on the command line is always searched
+  include: { type: 'string', multiple: true }, // only names matching one of these globs
+  exclude: { type: 'string', multiple: true }, // not names matching one of these globs
+  'changed-within': { type: 'string' }, // only files modified within 30m / 2h / 7d / 2w, since a date, today, ...
+  gitlog: { type: 'boolean', short: 'g', default: false }, // search git log's commits, one record each; auto-scope picks which
+  'auto-scope': { type: 'boolean', default: true }, // narrow those files by what Jev says a meaning restricts to; --no-auto-scope: don't
+  color: { type: 'string', default: 'auto' }, // auto / always / never
+  summarize: { type: 'string' }, // pipe what would print to this LLM CLI and print its answer instead
+  'summarize-prompt': { type: 'string' }, // the user's own instruction, added after the fixed one
+  format: { type: 'string', default: 'plain' }, // plain / markdown / html: --rank's output, or asked of the summarizer
+  template: { type: 'string' }, // --rank's or --summarize's --format=html document: a NAME under the templates dirs, or a file; list: the names
+  'install-templates': { type: 'boolean', default: false }, // copy the bundled templates to ~/.config/sys1grep/templates
+  cache: { type: 'string' }, // on|off|only: reuse Jev's stored answers (#252); clear: empty the cache
+  'summarize-format': { type: 'string' }, // removed (--format): parsed only to say so
+  // the API settings, each overriding its environment variable
+  'sys1-model': { type: 'string' }, // SYS1GREP_MODEL
+  'sys1-url': { type: 'string' }, // SYS1GREP_URL
+  'sys1-api-key': { type: 'string' }, // SYS1GREP_API_KEY / TYPESAFE_API_KEY
+  help: { type: 'boolean', short: 'h', default: false },
+  version: { type: 'boolean', short: 'V', default: false },
+  lang: { type: 'string' }, // en|ja: --help's language, and --serve's page default (SYS1GREP_LANG, settings.json lang)
+};
+// --no-rank / --no-summarize: parseArgs negates booleans only, so a bare one becomes a value no real argument can
+// hold (this NUL), cleared by the caller once parsed; the later of --no-rank/--rank=X (or --no-summarize/
+// --summarize=X) wins, as for any option.
+export const OFF = '\0';
+export const REVERSE_REMOVED = '--reverse was replaced by --step-reverse-to';
+// A bare --color / --dedup / --rank / --summarize / --cache means an implied default value, as grep's bare --color does;
+// parseArgs cannot express an optional value, so this fills it in before parseArgs ever sees it. --no-filename is
+// grep's name for --no-with-filename, --null-data grep's name for -z. summarizer: the TOOL a bare --summarize
+// fills in (sys1grep.mjs's own SYS1GREP_SUMMARIZER-or-"claude"; a caller that only validates shape, never runs the
+// result, may pass any placeholder).
+export const fill = (a, summarizer) => (a === '--color' ? '--color=auto' : a === '--null-data' ? '-z' : a === '--no-filename' ? '--no-with-filename'
+  : a === '--cache' ? '--cache=on' : a === '--summarize' ? `--summarize=${summarizer || 'claude'}` : a === '--dedup' ? '--dedup=always' : a === '--rank' ? '--rank=jev'
+  : a === '--no-rank' ? `--rank=${OFF}` : a === '--no-summarize' ? `--summarize=${OFF}` : a);
+// Parses a default-options array (SYS1GREP_OPTS / settings.json opts), already fill()ed, with sys1grep.mjs's own
+// parseArgs configuration, and refuses anything that is not a plain search option: a positional (a target or any
+// other stray value, #196), a meaning or expression (-e/-a/-v/-Q/--step-to go on the command line, not in
+// defaults), or a command (--install-templates, --template=list, --cache=clear: these do something instead of searching). Shared
+// by sys1grep.mjs's own gate and serve.mjs's settings-panel validation, so the two can never accept something the
+// other would refuse. Throws (a parseArgs error, or the refusal above) with sys1grep's own wording either way;
+// returns { tokens } — every parsed token, so a caller can read what option each one named (its canonical
+// OPTIONS key, however it was spelled: -r, --r, or inside a cluster like -rC1 or -nr) — on success.
+export function parseOpts(filledArgs) {
+  const { tokens } = parseArgs({ args: filledArgs, options: OPTIONS, allowPositionals: true, allowNegative: true, tokens: true });
+  const command = k => k.name === 'install-templates' || (k.name === 'template' && k.value === 'list') || (k.name === 'cache' && k.value === 'clear');
+  if (tokens.some(k => k.name === 'reverse')) throw new Error(REVERSE_REMOVED);
+  const bad = tokens.find(k => k.kind !== 'option' || ['e', 'P', 'a', 'v', 'question', 'step-to', 'step-reverse-to', 'cached', 'untracked'].includes(k.name) || command(k));
+  const reason = n => (n.startsWith('step-') ? 'expressions go on the command line' : n === 'P' ? 'a regex goes on the command line' : 'meanings go on the command line');
+  if (bad) throw new Error(bad.kind !== 'option' ? `'${bad.value ?? '--'}' is not an option` : command(bad) ? `${bad.rawName}${bad.value === undefined ? '' : `=${bad.value}`} is not allowed (it does something instead of searching)` : ['cached', 'untracked'].includes(bad.name) ? `--${bad.name} is not allowed (what is searched goes on the command line)` : `${bad.name.length > 1 ? '--' : '-'}${bad.name} is not allowed (${reason(bad.name)})`);
+  return { tokens };
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@uehaj/sys1grep",
-  "version": "0.5.0-next.1",
-  "description": "grep by meaning, across languages. TypeSafe Jev scores every line against a meaning; combine meanings with AND/OR/NOT.",
+  "version": "0.5.0",
+  "description": "grep by meaning, across languages. TypeSafe Jev scores each line (or record, sentence, function) against a meaning; combine meanings and regexes with AND/OR/NOT. Includes git sys1grep and a local search page (--serve).",
   "keywords": [
     "grep",
     "semantic",
@@ -9,7 +9,10 @@
     "typesafe",
     "jev",
     "cli",
-    "japanese"
+    "japanese",
+    "git",
+    "regex",
+    "llm"
   ],
   "license": "MIT",
   "author": "Junji UEHARA",
@@ -24,12 +27,25 @@
   },
   "files": [
     "sys1grep.mjs",
+    "serve.mjs",
+    "settings.mjs",
+    "options.mjs",
     "git-sys1grep.mjs",
+    "lib/sentences.mjs",
+    "examples/app.js",
+    "examples/app.py",
+    "examples/logs.txt",
+    "examples/prose.txt",
+    "examples/tickets.txt",
     "README.md",
     "README.ja.md",
     "docs/color.svg",
     "docs/demo.svg",
-    "docs/sentence.svg"
+    "docs/sentence.svg",
+    "templates/default.html",
+    "templates/print.html",
+    "templates/search.html",
+    "templates/terminal.html"
   ],
   "engines": {
     "node": ">=20.16"
@@ -39,6 +55,14 @@
     "test:offline": "sh tests/offline.sh",
     "judge": "node --no-warnings --env-file-if-exists=.env tests/judge.mts",
     "scope-eval": "node tests/scope-eval.mjs",
-    "release": "sh scripts/release.sh"
+    "dedup-eval": "node tests/dedup-eval.mjs",
+    "cause-eval": "node tests/cause-eval.mjs",
+    "skill-eval": "bash tests/skill-eval/run.sh",
+    "serve": "node sys1grep.mjs --serve",
+    "demo": "node scripts/demo-serve.mjs",
+    "serve:portless": "portless run --name sys1grep npm run serve:port --",
+    "serve:port": "node sys1grep.mjs --serve=$PORT",
+    "release": "sh scripts/release.sh",
+    "smoke": "node --no-warnings tests/smoke.mts"
   }
 }