@uehaj/sys1grep 0.5.0-next.2 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/sys1grep.mjs CHANGED
@@ -5,12 +5,16 @@
5
5
  // -e / -Q terms are OR'd; -a / -v attach AND / AND NOT to the preceding term: (A and B and not C) or D.
6
6
  // A leading ! negates just that meaning: -e A -e '!B' is A or not B.
7
7
  import { execFileSync, spawn, spawnSync } from 'node:child_process';
8
- import { constants, copyFileSync, existsSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, statSync, writeSync } from 'node:fs';
8
+ import { createHash } from 'node:crypto';
9
+ import { constants, copyFileSync, existsSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, renameSync, rmSync, statSync, writeFileSync, writeSync } from 'node:fs';
9
10
  import { gunzipSync } from 'node:zlib';
10
11
  import { homedir } from 'node:os';
11
12
  import { dirname, resolve } from 'node:path';
12
13
  import { fileURLToPath } from 'node:url';
13
14
  import { format, parseArgs } from 'node:util';
15
+ import { FIELDS, SETTINGS_FILE, SETTINGS_SHOWN, maskOpts, readSettings } from './settings.mjs';
16
+ import { OFF, OPTIONS, REVERSE_REMOVED, fill, parseOpts } from './options.mjs';
17
+ import { CJK, needsJudging, toSentences } from './lib/sentences.mjs';
14
18
 
15
19
  // Node 20 colors console.error red on a terminal (22 does not): stderr gets exactly what sys1grep writes.
16
20
  // A write error (EPIPE: the reader quit) is dropped, as console.error drops it, so it never turns into exit 2.
@@ -19,12 +23,26 @@ process.stderr.on('error', () => {});
19
23
  // Errors are one line plus exit code 2, like grep. No stack traces.
20
24
  const die = (msg, hint = true) => { console.error(`sys1grep: ${msg}${hint ? "\nTry 'sys1grep --help' for more information." : ''}`); process.exit(2); };
21
25
  process.on('uncaughtException', e => die(e.message));
26
+ // /pattern/flags (first and last char /, JS flags): a regex term's shape, used both by openStep (below, to decide
27
+ // whether a bare --step-to/--step-reverse-to value becomes -e or -P) and by the term parser further down.
28
+ const RE_SHAPE = /^\/(.*)\/([dgimsuvy]*)$/s;
29
+
30
+ // --serve [PORT] (#166): a search page on 127.0.0.1; serve.mjs runs this script again for each search.
31
+ if (process.argv.slice(2, process.argv.indexOf('--') < 0 ? undefined : process.argv.indexOf('--')).some(a => /^--serve(=|$)/.test(a))) {
32
+ (await import('./serve.mjs')).serve(process.argv.slice(2));
33
+ await new Promise(() => {}); // the server keeps the process alive; Ctrl-C ends it
34
+ }
22
35
 
23
- // Settings come from the environment; ~/.config/sys1grep/.env fills in what it lacks. Never ./.env: the current
24
- // directory may be an untrusted checkout, and its .env could point SYS1GREP_URL at a server that collects the key.
36
+ // Settings come from the environment, then ~/.config/sys1grep/settings.json, then ~/.config/sys1grep/.env. Never
37
+ // ./.env: the current directory may be an untrusted checkout, and its .env could point SYS1GREP_URL at a server that
38
+ // collects the key. settings.json wins over .env so that a value saved there takes effect.
25
39
  // For one minor release (removed in 1.0.0, see #93): fall back to the old SEMGREP_* names and
26
40
  // ~/.config/semgrep/.env, printing one deprecation line to stderr each time a fallback is actually used.
27
41
  const deprecated = (was, now) => console.error(`sys1grep: ${was} is deprecated; use ${now}`);
42
+ let settings;
43
+ try { settings = readSettings() ?? {}; } catch (e) { die(`${SETTINGS_SHOWN}: ${e.message}`, false); }
44
+ if (process.platform !== 'win32' && (settings.key || settings.summarizerKey) && statSync(SETTINGS_FILE).mode & 0o077)
45
+ console.error(`sys1grep: warning: ${SETTINGS_SHOWN} holds a key and others can read it; chmod 600 ${SETTINGS_SHOWN}`);
28
46
  // --verbose / --dry-run (#90) trace a setting's source back to here: a name in shellEnv came from the real
29
47
  // environment, one that only shows up after loadEnvFile came from envFile.
30
48
  const shellEnv = new Set(Object.keys(process.env));
@@ -34,157 +52,127 @@ let envFile = null;
34
52
  if (existsSync(newUserEnv)) { process.loadEnvFile(newUserEnv); envFile = newUserEnv; } // never overrides variables already set
35
53
  else if (existsSync(oldUserEnv)) { process.loadEnvFile(oldUserEnv); envFile = oldUserEnv; deprecated('~/.config/semgrep/.env', '~/.config/sys1grep/.env'); }
36
54
  const tildeEnvFile = envFile && envFile.replace(homedir(), '~');
37
- // A setting's source, for --verbose / --dry-run: the env var name that supplied it, plus the .env file
38
- // when it wasn't already in the real environment.
39
- const envLabel = name => (shellEnv.has(name) ? name : `${name}, ${tildeEnvFile}`);
40
- const fromEnv = name => { // { value, name }: SYS1GREP_<name>, falling back to SEMGREP_<name>
41
- const newName = `SYS1GREP_${name}`, oldName = `SEMGREP_${name}`;
42
- if (process.env[newName] !== undefined) return { value: process.env[newName], name: newName };
43
- if (process.env[oldName] !== undefined) { deprecated(oldName, newName); return { value: process.env[oldName], name: oldName }; }
44
- return { value: undefined, name: null };
55
+ // A setting's source, for --verbose / --dry-run: { value, name, file }. name: the env var or settings.json field that
56
+ // supplied it (null: none did); file: where it was written, null for the real environment.
57
+ // envLabel: "NAME" or "NAME, FILE"; keyLabel the same as "NAME (FILE)", for the key lines.
58
+ const envLabel = m => (m.file ? `${m.name}, ${m.file}` : m.name);
59
+ const keyLabel = m => (m.file ? `${m.name} (${m.file})` : m.name);
60
+ // The first set of SYS1GREP_<name>, SEMGREP_<name> (deprecated) and the other names given, as before settings.json;
61
+ // settings.json's field comes in only where that one came from the .env file or was not set at all. An env var set
62
+ // to '' counts as unset here (main's SYS1GREP_API_KEY || TYPESAFE_API_KEY did the same), so it falls through to the
63
+ // next name and then to settings.json, except OPTS: SYS1GREP_OPTS= is documented to still drop settings.json's opts.
64
+ const fromEnv = (name, ...more) => {
65
+ const field = Object.keys(FIELDS).find(f => FIELDS[f] === name);
66
+ const isSet = v => process.env[v] !== undefined && (field === 'opts' || process.env[v] !== '');
67
+ const n = [`SYS1GREP_${name}`, `SEMGREP_${name}`, ...more].find(isSet);
68
+ if (!shellEnv.has(n) && settings[field] !== undefined) return { value: settings[field], name: field, file: SETTINGS_SHOWN };
69
+ if (n === undefined) return { value: undefined, name: null, file: null };
70
+ if (n.startsWith('SEMGREP_')) deprecated(n, `SYS1GREP_${name}`);
71
+ return { value: process.env[n], name: n, file: shellEnv.has(n) ? null : tildeEnvFile };
45
72
  };
46
- const envURL = fromEnv('URL'), envMODEL = fromEnv('MODEL'), envAPI_KEY = fromEnv('API_KEY'), envOPTS = fromEnv('OPTS'),
47
- envSUMMARIZER = fromEnv('SUMMARIZER'), envSUMMARIZER_MODEL = fromEnv('SUMMARIZER_MODEL'), envSUMMARIZER_API_KEY = fromEnv('SUMMARIZER_API_KEY');
48
- const SYS1GREP_URL = envURL.value, SYS1GREP_MODEL = envMODEL.value, SYS1GREP_API_KEY = envAPI_KEY.value,
49
- SYS1GREP_OPTS = envOPTS.value ?? '', SYS1GREP_SUMMARIZER = envSUMMARIZER.value, SYS1GREP_SUMMARIZER_MODEL = envSUMMARIZER_MODEL.value;
50
- const { TYPESAFE_API_KEY } = process.env;
73
+ const envURL = fromEnv('URL'), envMODEL = fromEnv('MODEL'), envAPI_KEY = fromEnv('API_KEY', 'TYPESAFE_API_KEY'), envOPTS = fromEnv('OPTS'),
74
+ envSUMMARIZER = fromEnv('SUMMARIZER'), envSUMMARIZER_MODEL = fromEnv('SUMMARIZER_MODEL'), envSUMMARIZER_API_KEY = fromEnv('SUMMARIZER_API_KEY'),
75
+ envLANG = fromEnv('LANG');
76
+ const SYS1GREP_URL = envURL.value, SYS1GREP_MODEL = envMODEL.value,
77
+ SYS1GREP_SUMMARIZER = envSUMMARIZER.value, SYS1GREP_SUMMARIZER_MODEL = envSUMMARIZER_MODEL.value;
78
+ // The default options: SYS1GREP_OPTS split on spaces, or settings.json's opts as they are. OPTS_NAME names them in
79
+ // messages and in --verbose's "(SOURCE)" tags.
80
+ const OPTS_NAME = envOPTS.file === SETTINGS_SHOWN ? 'settings.json opts' : 'SYS1GREP_OPTS';
81
+ const optsArgs = Array.isArray(envOPTS.value) ? envOPTS.value : (envOPTS.value ?? '').split(/\s+/).filter(Boolean);
51
82
 
52
- // A bare --color means --color=auto (as in grep); parseArgs cannot express an optional value, so fill it in first.
53
- // --no-filename is grep's name for --no-with-filename, --null-data grep's name for -z.
54
- // --no-rank / --no-summarize: parseArgs negates booleans only, so they become a value no argument can hold (a NUL),
55
- // cleared below; the later one wins, as for any option.
56
- const OFF = '\0';
57
- const fill = a => (a === '--color' ? '--color=auto' : a === '--null-data' ? '-z' : a === '--no-filename' ? '--no-with-filename'
58
- : a === '--summarize' ? `--summarize=${SYS1GREP_SUMMARIZER || 'claude'}` : a === '--dedup' ? '--dedup=always' : a === '--rank' ? '--rank=jev'
59
- : a === '--no-rank' ? `--rank=${OFF}` : a === '--no-summarize' ? `--summarize=${OFF}` : a);
60
- const OPTIONS = {
61
- e: { type: 'string', multiple: true },
62
- a: { type: 'string', multiple: true },
63
- v: { type: 'string', multiple: true },
64
- question: { type: 'string', multiple: true, short: 'Q' },
65
- level: { type: 'string', default: 'normal' }, // strictness preset: loose / normal / strict
66
- r: { type: 'boolean', default: false }, // recurse into directories
67
- cached: { type: 'boolean', default: false }, // git sys1grep only: search the index instead of the working tree
68
- untracked: { type: 'boolean', default: false }, // git sys1grep only: also search untracked files (.gitignore still applies)
69
- l: { type: 'boolean', default: false }, // print only matching file names
70
- 'with-filename': { type: 'boolean', short: 'H' }, // prefix file names even for one file; --no-filename: never
71
- t: { type: 'string' }, // positive threshold: match when p >= t (default from preset)
72
- T: { type: 'string' }, // negative threshold: "not X" when p < T (default from preset)
73
- chunk: { type: 'string', default: '30' }, // lines per request
74
- c: { type: 'boolean', default: false }, // count of matching lines per file (grep -c)
75
- quiet: { type: 'boolean', short: 'q', default: false }, // print nothing, exit status only (grep -q)
76
- j: { type: 'string', default: '8' }, // concurrent requests
77
- A: { type: 'string' }, // N lines of trailing context
78
- B: { type: 'string' }, // N lines of leading context
79
- C: { type: 'string' }, // N lines of context on both sides
80
- n: { type: 'boolean', default: false }, // line numbers
81
- z: { type: 'boolean', default: false }, // records are NUL-terminated, on input and output (grep -z); --unit=zero
82
- unit: { type: 'string', default: 'line' }, // line / zero / sentence-by-jev / sentence-by-rule
83
- o: { type: 'boolean', default: false }, // with --unit=sentence-by-*, print only the matching sentences (grep -o)
84
- p: { type: 'boolean', default: false }, // print each meaning's probability
85
- dedup: { type: 'string', default: 'never' }, // auto|always|never: judge one representative per template, reuse its answer
86
- rank: { type: 'string' }, // jev|match: print the results best first, scored by Jev's answer on each or by its best match
87
- // multi-step matching (#163): the expression before --step-to finds the start, the one after it the end
88
- 'step-to': { type: 'boolean', multiple: true },
89
- edges: { type: 'string' }, // FILE: the edges to walk, one per line; default: the calls between functions
90
- reverse: { type: 'boolean', default: false }, // walk the edges backwards
91
- hops: { type: 'string', default: '0..' }, // N, M..N or M..: the hops an end may be at
92
- 'dry-run': { type: 'boolean', default: false }, // print the files and requests, send nothing
93
- verbose: { type: 'boolean', default: false }, // print the files and requests to stderr while searching
94
- interactive: { type: 'boolean', short: 'i', default: false }, // show what --dry-run would send, search on a yes
95
- yes: { type: 'boolean', short: 'y', default: false }, // skip the size/cost guard's question, answering yes (#58)
96
- 'max-columns': { type: 'string', short: 'M' }, // a unit past this many characters is skipped (rg's -M/--max-columns)
97
- 'max-filesize': { type: 'string' }, // a target past this size (K/M/G) is listed and confirmed (rg's name)
98
- 'max-cost': { type: 'string', default: '1' }, // ask before sending when the estimated price is over this many USD
99
- // which files -r finds and git sys1grep lists; with -r a file named on the command line is always searched
100
- include: { type: 'string', multiple: true }, // only names matching one of these globs
101
- exclude: { type: 'string', multiple: true }, // not names matching one of these globs
102
- 'changed-within': { type: 'string' }, // only files modified within 30m / 2h / 7d / 2w, since a date, today, ...
103
- gitlog: { type: 'boolean', short: 'g', default: false }, // search git log's commits, one record each; auto-scope picks which
104
- 'auto-scope': { type: 'boolean', default: true }, // narrow those files by what Jev says a meaning restricts to; --no-auto-scope: don't
105
- color: { type: 'string', default: 'auto' }, // auto / always / never
106
- summarize: { type: 'string' }, // pipe what would print to this LLM CLI and print its answer instead
107
- 'summarize-prompt': { type: 'string' }, // the user's own instruction, added after the fixed one
108
- format: { type: 'string', default: 'plain' }, // plain / markdown / html: --rank's output, or asked of the summarizer
109
- template: { type: 'string' }, // --rank's or --summarize's --format=html document: a NAME under the templates dirs, or a file; list: the names
110
- 'install-templates': { type: 'boolean', default: false }, // copy the bundled templates to ~/.config/sys1grep/templates
111
- 'summarize-format': { type: 'string' }, // removed (--format): parsed only to say so
112
- // the API settings, each overriding its environment variable
113
- 'sys1-model': { type: 'string' }, // SYS1GREP_MODEL
114
- 'sys1-url': { type: 'string' }, // SYS1GREP_URL
115
- 'sys1-api-key': { type: 'string' }, // SYS1GREP_API_KEY / TYPESAFE_API_KEY
116
- help: { type: 'boolean', short: 'h', default: false },
117
- version: { type: 'boolean', short: 'V', default: false },
118
- };
119
83
  // SYS1GREP_OPTS holds default options only: no meanings, no files, no --. It goes in front of the arguments, so the
120
- // command line wins (a later value counts; --no-X clears a flag).
121
- const defaults = SYS1GREP_OPTS.split(/\s+/).filter(Boolean).map(fill);
84
+ // command line wins (a later value counts; --no-X clears a flag). fill/OFF: options.mjs (shared with serve.mjs).
85
+ const defaults = optsArgs.map(a => fill(a, SYS1GREP_SUMMARIZER));
122
86
  // --step-to X (#163) is --step-to -e X, and so is --step-to=X; a bare --step-to (followed by an option) opens the end
123
- // expression for the -e / -a / -v / -Q after it. After --, every argument is a file. X, and the MEANING of -e / -a /
124
- // -v / -Q, may start with a dash ("--summarize hands the lines on"): an option holds no space, and is -- and a word or
125
- // a known short one. parseArgs refuses a separate value starting with -, so such a value is attached.
87
+ // expression for the -e / -a / -v / -Q after it; the same for --step-reverse-to. After --, every argument is a file.
88
+ // X, and the MEANING of -e / -a / -v / -Q, may start with a dash ("--summarize hands the lines on"): an option holds
89
+ // no space, and is -- and a word or a known short one. parseArgs refuses a separate value starting with -, so such a value is attached.
126
90
  const SHORTS = new Set(Object.entries(OPTIONS).map(([k, o]) => o.short ?? (k.length === 1 ? k : null)).filter(Boolean));
127
91
  const isOption = a => !/\s/.test(a) && (a.startsWith('--') || (/^-[^-]/.test(a) && SHORTS.has(a[1])));
128
92
  const openStep = args => {
129
93
  const out = [];
130
94
  for (let i = 0; i < args.length; i++) {
131
- const m = !out.includes('--') && /^--step-to(?:=([\s\S]*))?$/.exec(args[i]);
95
+ const m = !out.includes('--') && /^(--step(?:-reverse)?-to)(?:=([\s\S]*))?$/.exec(args[i]);
132
96
  const dashed = i + 1 < args.length && args[i + 1].startsWith('-') && !isOption(args[i + 1]);
133
- if (!out.includes('--') && dashed && ['-e', '-a', '-v', '-Q', '--question'].includes(args[i])) { out.push(`${args[i]}${args[i] === '--question' ? '=' : ''}${args[++i]}`); continue; }
97
+ if (!out.includes('--') && dashed && ['-e', '-P', '-a', '-v', '-Q', '--question'].includes(args[i])) { out.push(`${args[i]}${args[i] === '--question' ? '=' : ''}${args[++i]}`); continue; }
134
98
  if (!m) { out.push(args[i]); continue; }
135
- const value = m[1] ?? (i + 1 < args.length && !isOption(args[i + 1]) ? args[++i] : undefined);
136
- if (value !== undefined && !value.trim()) die('--step-to: MEANING must not be empty');
137
- out.push('--step-to', ...(value === undefined ? [] : [`-e${value}`]));
99
+ const value = m[2] ?? (i + 1 < args.length && !isOption(args[i + 1]) ? args[++i] : undefined);
100
+ if (value !== undefined && !value.trim()) die(`${m[1]}: MEANING must not be empty`);
101
+ // A bare end/start expression right after --step-to / --step-reverse-to is -e MEANING, or -P '/RE/' when it is
102
+ // shaped like a regex (-e no longer accepts one): --step-to '/raise /' is --step-to -P '/raise /'. A bare
103
+ // !/RE/ is --step-to -v '/RE/' (-P never takes a leading !, and -v is already "not" at the front); a bare
104
+ // !MEANING is still -e !MEANING, -e's own per-literal negation.
105
+ const bang = value !== undefined && value.startsWith('!');
106
+ const bare = bang ? value.slice(1) : value;
107
+ const isRe = value !== undefined && RE_SHAPE.test(bare);
108
+ out.push(m[1], ...(value === undefined ? [] : [`${isRe ? (bang ? '-v' : '-P') : '-e'}${isRe ? bare : value}`]));
138
109
  }
139
110
  return out;
140
111
  };
141
112
  let optsInteractive = false; // -i from SYS1GREP_OPTS: a script without a terminal is told where it came from
142
113
  try {
143
- const { tokens: t } = parseArgs({ args: defaults, options: OPTIONS, allowPositionals: true, allowNegative: true, tokens: true });
114
+ const { tokens: t } = parseOpts(defaults);
144
115
  optsInteractive = t.some(k => k.name === 'interactive' && !k.rawName.startsWith('--no-'));
145
- const command = k => k.name === 'install-templates' || (k.name === 'template' && k.value === 'list');
146
- const bad = t.find(k => k.kind !== 'option' || ['e', 'a', 'v', 'question', 'step-to', 'cached', 'untracked'].includes(k.name) || command(k));
147
- if (bad) die(`SYS1GREP_OPTS: ${bad.kind !== 'option' ? `'${bad.value ?? '--'}' is not an option` : command(bad) ? `${bad.rawName}${bad.value === undefined ? '' : `=${bad.value}`} is not allowed (it does something instead of searching)` : ['cached', 'untracked'].includes(bad.name) ? `--${bad.name} is not allowed (what is searched goes on the command line)` : `${bad.name.length > 1 ? '--' : '-'}${bad.name} is not allowed (${bad.name === 'step-to' ? 'expressions' : 'meanings'} go on the command line)`}`);
148
- } catch (e) { die(`SYS1GREP_OPTS: ${e.message}`); }
116
+ } catch (e) { die(`${OPTS_NAME}: ${e.message}`); }
149
117
  const { values: opt, positionals: files, tokens } = parseArgs({
150
- args: [...defaults, ...openStep(process.argv.slice(2)).map(fill)],
118
+ args: [...defaults, ...openStep(process.argv.slice(2)).map(a => fill(a, SYS1GREP_SUMMARIZER))],
151
119
  options: OPTIONS,
152
120
  allowPositionals: true,
153
121
  allowNegative: true,
154
122
  tokens: true,
155
123
  });
156
124
  if (opt['summarize-format'] !== undefined) die('--summarize-format was removed; use --format', false);
125
+ if (opt.reverse !== undefined) die(REVERSE_REMOVED, false);
157
126
  for (const k of ['rank', 'summarize']) if (opt[k] === OFF) delete opt[k];
127
+ // #229: --summarize=ask prints the matches, then asks whether to summarize them, with the TOOL a bare --summarize uses
128
+ const askSummary = opt.summarize === 'ask';
129
+ if (askSummary) opt.summarize = SYS1GREP_SUMMARIZER || 'claude';
130
+ // --lang=en|ja (#270): --help's language, and (via SYS1GREP_PROBE below) --serve's page default. langValue is the
131
+ // resolved value before any locale fallback, undefined when nothing set it.
132
+ const langValue = opt.lang ?? envLANG.value;
133
+ if (langValue !== undefined && !['en', 'ja'].includes(langValue))
134
+ die(`${opt.lang !== undefined ? '--lang' : envLabel(envLANG)}: '${langValue}' is not en or ja`);
135
+ // #222: --serve's launch check. With the options settled (command line, SYS1GREP_OPTS, settings.json opts) and before
136
+ // any check of how they combine, say whether a folder target would be searched: -r in effect, or -g / git sys1grep,
137
+ // which read no target files. Nothing is read or sent.
138
+ if (process.env.SYS1GREP_PROBE === 'recursive') {
139
+ // url: the endpoint a search sends to (customUrl, below), null for the default; --serve asks a local one whether it is the fake
140
+ console.log(JSON.stringify({ r: !!opt.r, g: !!opt.gitlog, git: globalThis.SYS1GREP_GIT === true, url: opt['sys1-url'] || SYS1GREP_URL || null, lang: langValue ?? null }));
141
+ process.exit(0);
142
+ }
158
143
  // --verbose / --dry-run (#90): where a parsed option's value came from. null: never set (caller says "default").
159
144
  // '': the command line (no source shown, as for any setting not from an env var or SYS1GREP_OPTS). 'SYS1GREP_OPTS':
160
145
  // its last token is one of the `defaults` this run prepended.
161
146
  const optSrc = name => {
162
147
  const last = tokens.filter(k => k.kind === 'option' && k.name === name).at(-1);
163
- return !last ? null : last.index < defaults.length ? 'SYS1GREP_OPTS' : '';
148
+ return !last ? null : last.index < defaults.length ? OPTS_NAME : '';
164
149
  };
165
150
  // --help: Japanese when the locale starts with ja, English otherwise
166
- const HELP_EN = `usage: sys1grep [OPTION]... -e MEANING|-Q QUESTION [-a MEANING] [-v MEANING]... [FILE...]
167
- sys1grep [OPTION]... EXPRESSION --step-to EXPRESSION [FILE...]
151
+ const HELP_EN = `usage: sys1grep [OPTION]... -e MEANING|-Q QUESTION|-P /RE/ [-a MEANING] [-v MEANING]... [FILE...]
152
+ sys1grep [OPTION]... -e START1 [-e START2]... --step-to END1 [-e END2]... [FILE...]
168
153
  grep by meaning, powered by Jev (TypeSafe System One). Reads stdin when FILE is omitted.
169
154
  As git sys1grep, FILE arguments are pathspecs and every tracked file is searched, like git grep.
170
155
 
171
- -e MEANING lines matching this meaning (several -e are OR'd)
156
+ -e MEANING lines matching this meaning (several -e are OR'd). Always a meaning: a value shaped
157
+ /RE/FLAGS is refused, naming -P (e.g. -e '/catch/' is no longer a regex: use -P '/catch/').
158
+ A meaning that really starts and ends with / gets through with a leading space (-e ' /cat/')
172
159
  -Q, --question QUESTION lines that answer QUESTION, not lines asking it; the same as
173
160
  -e "the line answers: QUESTION". "why the job failed" matches "the disk was full",
174
- "whether the server is down" matches a denial too. Combines with -e / -a / -v / ! like -e
175
- -a MEANING AND onto the preceding -e/-Q term. -e A -a B -e C = (A and B) or C
176
- -v MEANING AND NOT onto the preceding -e/-Q term. -e A -v B = A and not B
177
- At the front it is a bare negation. -v B = not B (like grep -v)
178
- !MEANING a leading ! negates just that meaning, in -e / -Q / -a / -v alike
179
- -e A -e '!B' = A or not B. -a '!C' is the same as -v C
180
- /RE/FLAGS a regex term (first and last char /, JS flags dgimsuvy): matched locally, no
181
- request. Prefilters its AND term: only units it holds for ask that term's meanings
182
- -e '/ERROR|FATAL/' -a '/timeout/i' -v '/^DEBUG/' !/RE/ negates it, as above
161
+ "whether the server is down" matches a denial too. Combines with -e / -a / -v / ! like -e.
162
+ Always a question, never a regex
163
+ -P /RE/FLAGS a regex term (first and last char /, JS flags dgimsuvy), OR'd like -e; matched
164
+ locally, no request. Prefilters its AND term: only units it holds for ask that
165
+ term's meanings. -P '/ERROR|FATAL/' -a '/timeout/i' -v '/^DEBUG/'
183
166
  named/numbered groups pass to that term's meanings as $<name>, $1-$99, $&, $$
184
167
  (ECMAScript's replace patterns). $<name> naming no group, or a negated regex's
185
168
  group, is an error; prefer $<name> and single quotes ($1 doesn't survive double
186
- quotes in sh/bash/zsh). A meaning that really starts and ends with / needs a
187
- leading space to not be read as a regex
169
+ quotes in sh/bash/zsh). -P '!/RE/' is not allowed; negate with -v '/RE/'
170
+ -a MEANING AND onto the preceding -e/-Q/-P term. -e A -a B -e C = (A and B) or C
171
+ -v MEANING AND NOT onto the preceding -e/-Q/-P term. -e A -v B = A and not B
172
+ At the front it is a bare negation. -v B = not B (like grep -v)
173
+ -a and -v also take a /RE/ (a regex AND / AND NOT: -P '/RE/' -a '/one/' -v '/other/')
174
+ !MEANING a leading ! negates just that meaning, in -e / -Q / -a / -v alike (not -P)
175
+ -e A -e '!B' = A or not B. -a '!C' is the same as -v C
188
176
  --level=LEVEL strictness preset, sets both thresholds (default normal)
189
177
  loose : -t 0.3 -T 0.7 catch more, accept some noise
190
178
  normal : -t 0.5 -T 0.5
@@ -195,9 +183,10 @@ As git sys1grep, FILE arguments are pathspecs and every tracked file is searched
195
183
  -r recurse into directories (current directory when FILE is omitted). Skips .git,
196
184
  node_modules, .ssh/.aws/.gnupg/.kube/.docker, binary files, likely secrets (.env*,
197
185
  .netrc, .npmrc, .git-credentials, *.pem, *.key, id_rsa*...) and generated files
198
- (*.map, *.min.js, *.min.css, package-lock.json and other lock files). Every searched
199
- line is sent to the TypeSafe API. Inside a git repository, what git ignores (.gitignore)
200
- is skipped too; a file or directory named on the command line is searched even so.
186
+ (*.map, *.min.js, *.min.css, package-lock.json and other lock files), symlinks and
187
+ settings.json itself. Every searched line is sent to SYS1GREP_URL. Inside a git repository,
188
+ what git ignores (.gitignore) is skipped too; a file or directory named on the command line
189
+ is searched even so.
201
190
  A .gz file (a rotated log) is read decompressed, as zgrep does, and printed by its name
202
191
  --cached git sys1grep only: search the blobs staged in the index instead of the working tree, as
203
192
  git grep --cached. A file deleted from the working tree but still staged is still found.
@@ -218,7 +207,8 @@ As git sys1grep, FILE arguments are pathspecs and every tracked file is searched
218
207
  * also matches a leading dot (*.md matches .notes.md), as in rg --glob
219
208
  --changed-within=WHEN with -r and git sys1grep, only files modified within WHEN: 30m, 2h, 7d, 2w;
220
209
  since a date or time (2026-09-01 is local midnight, 2026-09-01T09:00, ...Z); or today,
221
- this-week (from Monday) or this-month, in local time. By mtime, not git history
210
+ this-week (from Monday) or this-month, in local time. By mtime, not git history, so not with
211
+ --cached or a <tree>
222
212
  --no-auto-scope do not narrow the files -r and git sys1grep find by what a meaning says about them. By default
223
213
  each meaning first asks Jev, in one small request, whether it restricts its matches to a
224
214
  language or format (Python files, YAML files, ...) or to what changed within a span (the last
@@ -226,19 +216,20 @@ As git sys1grep, FILE arguments are pathspecs and every tracked file is searched
226
216
  month, the last year, this fiscal year) or to a place (test code, migrations, the README, the
227
217
  changelog, documentation, source code, logs), a git state (uncommitted, staged, untracked, this
228
218
  branch, not pushed, mine) or an author (the 30 with the most commits); a yes at 0.6 or more
229
- searches only those files. In a repository a time goes by commits: a committed file needs a
230
- commit since then, an uncommitted one its mtime
231
- (*.py *.pyi *.pyw; modified since then). Jev reads the whole meaning, so "案A、B、Cで" is not
232
- about C files. Per term: -e A -e B still searches B in the files A leaves out. Each scope goes
233
- to stderr as sys1grep: scope: ... (with --verbose, also the scopes each matching file got through
234
- and the files left out); with -r a file named on the command line is never narrowed,
235
- git sys1grep's pathspecs are narrowed like the rest (as with --include)
219
+ searches only those files (*.py *.pyi *.pyw; modified since then). In a repository a time goes
220
+ by commits: a committed file needs a commit since then, an uncommitted one its mtime. Jev reads the
221
+ whole meaning, so "案A、B、Cで" is not about C files. Per term: -e A -e B still searches B in the
222
+ files A leaves out. Each scope goes to stderr as sys1grep: scope: ... (with --verbose, also the
223
+ scopes each matching file got through and the files left out); with -r a file named on the
224
+ command line is never narrowed, git sys1grep's pathspecs are narrowed like the rest (as with
225
+ --include)
236
226
  --auto-scope turn it back on after --no-auto-scope in SYS1GREP_OPTS
237
227
  -g, --gitlog search the commits of git log, one record each (hash, date, subject, body), instead of
238
- files. With one term, auto-scope becomes git log's arguments: a time --since (and --until
239
- when the span has an end: yesterday, last week, last month), a language
240
- pathspecs, an author or mine --author, this branch or not pushed a range; the command goes
241
- to stderr. FILE arguments are pathspecs. Not with -r or git sys1grep
228
+ files. With one term, auto-scope becomes git log's arguments: a time --since-as-filter
229
+ (--since on git before 2.37, which stops at the first older commit; and --until when
230
+ the span has an end: yesterday, last week, last month), a language pathspecs, an author or
231
+ mine --author, this branch or not pushed a range; the command goes to stderr. FILE
232
+ arguments are pathspecs. Not with -r or git sys1grep
242
233
  sys1grep -g -e 'a performance fix to the .mjs files today'
243
234
  -l print only the names of files with a match, not the lines
244
235
  -H, --with-filename prefix each line (and -c count) with its file name, even for a single file
@@ -248,10 +239,11 @@ As git sys1grep, FILE arguments are pathspecs and every tracked file is searched
248
239
  -C NUM print NUM lines of context before and after (-A NUM -B NUM)
249
240
  -c print only a count of matching lines per file (like grep -c)
250
241
  -q, --quiet print nothing, stop at the first match; exit 0 on a match, even after an error (like grep -q)
251
- --chunk=LINES lines per request (default 30)
242
+ --chunk=LINES lines per request (default 30; a request also carries at most 64 questions, so fewer lines
243
+ with 3 or more meanings)
252
244
  Lines in one request are each other's context, so a small chunk changes verdicts
253
245
  on ambiguous lines, not just speed
254
- -j N concurrent requests (default 8)
246
+ -j N concurrent requests (default 8; 1 for a local URL: localhost, 127.0.0.1, [::1])
255
247
  -n print line numbers
256
248
  --unit=UNIT the unit of judgement: line (default), zero (NUL-terminated record), sentence-by-jev / sentence-by-rule,
257
249
  or function.
@@ -275,12 +267,14 @@ As git sys1grep, FILE arguments are pathspecs and every tracked file is searched
275
267
  line before the next, as git grep -W: the diff=<driver> attribute and diff.<driver>.xfuncname
276
268
  pick the funcname lines, else sys1grep's rule for .js/.ts/.py, else a line starting with a
277
269
  letter, _ or $. -M defaults to 8000. Not with -z, -g or -o
278
- --step-to EXPRESSION multi-step matching: the expression before --step-to finds the start units; walk the
279
- edges from them breadth first, and print the paths to the reached units that match the expression
280
- after it (the ends). A step is a search, not a hop: every -e / -a / -v / -Q after --step-to is the
281
- end expression. --step-to MEANING or --step-to '/RE/' is --step-to -e MEANING. No score steers the
282
- walk; the units reached within --hops are judged against the end expression in one batch. A
283
- regex-only start and end send nothing.
270
+ --step-to END1 [-e END2]... multi-step matching, -e START1 [-e START2]... --step-to END1 [-e END2]...: find the
271
+ units (functions, by default) that match START, walk the edges (calls, or --edges) from them breadth
272
+ first, and print each path to a reached unit that matches END (an end): two searches (steps) joined
273
+ by any number of calls (hops). Each side is an expression of its own, and several -e on a side are
274
+ OR'd. A step is a search, not a hop: every -e / -P / -a / -v / -Q after --step-to is the end expression.
275
+ --step-to MEANING is --step-to -e MEANING; --step-to '/RE/' is --step-to -P '/RE/'. No score
276
+ steers the walk; the units reached within --hops are judged against the end expression in one
277
+ batch. A regex-only start and end send nothing.
284
278
  Without --edges the unit is a function (--unit=function) and an edge is a call: name( in a
285
279
  function's body (comments, docstrings and strings left out) links to every function of that name.
286
280
  Each path prints as a tree: the hop, file:line and the function's name, : for an end and - for a
@@ -288,11 +282,20 @@ As git sys1grep, FILE arguments are pathspecs and every tracked file is searched
288
282
  and why the walk stopped. Not with -z, -g, -o, -c, -l, -A/-B/-C, --rank, --summarize or --dedup.
289
283
  --max-cost asks again before the end's requests, counting what the start sent; --dry-run walks
290
284
  from every unit the start expression could hold for, a bound
285
+ Examples:
286
+ # where --summarize hands lines to the tool, and what a function it reaches does when the tool fails
291
287
  sys1grep -e "--summarize hands the lines to the tool" --step-to "what happens when it fails" sys1grep.mjs
288
+ # regex only, nothing is sent: main itself (hop 0) and the functions it reaches that hold a raise
289
+ sys1grep -P '/^ *def main/' --step-to -P '/raise /' *.py
290
+ # backwards: does main reach helper? (--step-reverse-to walks callee to caller)
291
+ sys1grep -P '/^ *def helper/' --step-reverse-to -P '/^ *def main/' *.py
292
+ # only ends exactly 1 or 2 calls from the start
293
+ sys1grep -P '/^ *def main/' --hops=1..2 --step-to -P '/raise /' *.py
292
294
  --edges=FILE the edges to walk instead of calls, one per line: FROM_FILE:LINE<TAB>FROM_NAME<TAB>TO_FILE:LINE
293
295
  <TAB>TO_NAME. A line stands for the unit holding it, of any --unit but zero. Any relation works
294
296
  (imports, links, log ids); write an undirected one both ways
295
- --reverse walk the edges backwards (callee to caller)
297
+ --step-reverse-to END1 [-e END2]... --step-to walking the edges backwards (callee to caller). Not with
298
+ --step-to
296
299
  --hops=N|M..N|M.. the hops an end may be at: exactly N, M to N, or M and more (default 0..). A unit's hop is
297
300
  its shortest distance from any start; hop 0 is a start itself
298
301
  -o print only what matched, one per line: each regex match, as grep -o (a line only meanings
@@ -304,36 +307,48 @@ As git sys1grep, FILE arguments are pathspecs and every tracked file is searched
304
307
  command line, then each file searched (units, and how many would be sent) and each request
305
308
  with its questions, grouped by wording (line ids read Lnnn). The key's value never prints.
306
309
  The --dedup and --unit=sentence-by-jev questions are answered no, so their counts are an estimate.
307
- The last line estimates the input tokens and, for TypeSafe itself, the price (~, within about 10%)
310
+ The last line estimates the input tokens and the price (~, within about 10%; --max-cost says whose)
308
311
  --verbose print the same to stderr while searching, and the summary line even when not a terminal
309
312
  -i, --interactive first show what --dry-run would send (files, lines, requests) and ask on the
310
313
  terminal; search only on y. Nothing is sent before the answer; no terminal is an error
311
314
  -M NUM, --max-columns=NUM send at most the first NUM characters of a line (or record, with -z); it
312
315
  is still searched and judged, but a match past NUM cannot be found there. Default 2000
313
- (8000 with -z)
316
+ (8000 with -z or --unit=function)
314
317
  --max-filesize=SIZE before any target is read, every one is sized (K/M/G suffix, default 10M); one
315
318
  over this is skipped outright and named on stderr, like rg's own --max-filesize. -y does
316
319
  not affect it (a named file over the limit is skipped too). stdin is sized once it is
317
320
  read, and skipped the same way if it is over. --cached / <tree>: targets are blobs, sized
318
- from their content too. a .gz is
319
- sized decompressed (zlib stops at the limit). -g's commits stay out of this: each is already bounded by -M
320
- when sent
321
- --max-cost=USD the input tokens about to be sent are estimated and priced; over this (default 1) the
321
+ from their content too. A .gz is sized decompressed (zlib stops at the limit). -g's commits
322
+ stay out of this: each is already bounded by -M when sent
323
+ --max-cost=USD the input tokens about to be sent are estimated and priced (a local URL is free; jev-latest 0.042,
324
+ clef 0.24, clef-flash 0.09 USD per M tokens; any other model as Jev); over this (default 1) the
322
325
  run asks to continue, on the terminal. No terminal and the limit exceeded is exit 2,
323
326
  naming the option that would let it through; -q does not change this (scripts pass -y).
324
327
  -i already asks unconditionally and earlier, so this does not ask again
325
328
  -y, --yes answer that question yes without asking (SYS1GREP_OPTS='--max-cost ... -y' for scripts)
329
+ --cache[=on|off|only] keep Jev's answers in ~/.config/sys1grep/cache, keyed by what is sent (model, lines,
330
+ questions) and the endpoint's host, so the same request again is answered from there: no
331
+ request, no cost. Default off (SYS1GREP_CACHE): the cache holds the lines searched. Bare
332
+ --cache is on; only answers from the cache alone: a request not there is exit 2, naming that
333
+ request, with nothing sent (unless -q already found a match). No expiry: Jev drifts about 0.05 between
334
+ runs, a cached answer is one of them. --dry-run says how many requests are cached and how many
335
+ to send. --cache=clear empties it; it must stand alone, and is refused in SYS1GREP_OPTS
326
336
  --dedup[=auto|always|never] judge one line per template instead of every line. Lines that differ only in
327
337
  ids, hashes, numbers, dates and times, paths and URLs share a template; one of them is sent and
328
338
  its answer is reused for the rest. Which of those may be folded depends on the meaning: a number
329
- decides "disk usage above 90%", a time decides "happened at night". Jev is asked first, one small
330
- request per meaning, and the kinds that could change a match are kept apart. Built for
339
+ decides "disk usage above 90%", a time decides "happened at night". always asks Jev first, one
340
+ small request per meaning, and keeps apart the kinds that could change a match. auto asks
341
+ nothing: it judges up to 5 sampled members of each template (the lowest, highest and middle of
342
+ each varying value). When their verdicts agree, the members not sampled share the first
343
+ sample's answers and the samples keep their own; when they differ, the rest are sent after
344
+ --max-cost is checked again (a few percent more than never). A value only between the samples
345
+ is missed, and --dry-run assumes every sample agrees, so its count is the floor. Built for
331
346
  machine-generated logs, where it can cut the cost by a factor of 30 or more; prose has no
332
347
  shared skeleton and barely folds, and a meaning that reads a value folds little.
333
348
  never (default, for now: see #143) never asks or folds, but still estimates locally, before
334
349
  anything is sent, what folding every kind (the best case) would save; when that would be at
335
350
  least twice the cost of --dedup's own question, one stderr hint names --dedup=auto (not with -q).
336
- auto asks and folds only when that estimate says it pays; bare --dedup is --dedup=always, which
351
+ auto folds only when that estimate says it pays; bare --dedup is --dedup=always, which
337
352
  always asks and folds, as every --dedup did before this option took a value. --verbose /
338
353
  --dry-run print the decision and its numbers for every value.
339
354
  With --unit=zero or --unit=sentence-by-* the unit that folds is the record or the sentence
@@ -352,25 +367,45 @@ As git sys1grep, FILE arguments are pathspecs and every tracked file is searched
352
367
  --summarize[=TOOL] pipe what would print (file names, -n, -A/-B/-C, -p) to TOOL, asked to summarize it as it
353
368
  bears on the meanings, and print TOOL's answer instead. TOOL: claude (default, or SYS1GREP_SUMMARIZER),
354
369
  run as claude -p --model haiku with no tools and no settings; llm (Simon Willison's, likewise no
355
- tools); pi (--no-tools and every --no-session/--no-context-files/--no-extensions/--no-skills/
356
- --no-prompt-templates); ollama or lmstudio (a local OpenAI-compatible server, no CLI); or an
357
- http(s):// URL, the same
358
- request to any OpenAI-compatible server (llama.cpp, vLLM, LocalAI, a gateway). ollama, lmstudio and a
359
- URL need SYS1GREP_SUMMARIZER_MODEL: none has a default model. The matching lines are sent a second
360
- time, to TOOL's provider (nowhere a second time with ollama, lmstudio or a local URL). No match runs
361
- nothing (exit 1); TOOL failing is exit 2. Not with -q, -l, -c.
370
+ tools); pi (--no-tools, --no-approve, --thinking off and every --no-session/--no-context-files/
371
+ --no-extensions/--no-skills/--no-prompt-templates); ollama or lmstudio (a local OpenAI-compatible
372
+ server, no CLI); or an http(s):// URL, the same request to any OpenAI-compatible server
373
+ (llama.cpp, vLLM, LocalAI, a gateway). ollama, lmstudio and a URL need SYS1GREP_SUMMARIZER_MODEL:
374
+ none has a default model. The matching lines are sent a second time, to TOOL's provider
375
+ (nowhere a second time with ollama, lmstudio or a local URL). No match runs nothing (exit 1);
376
+ TOOL failing is exit 2. Not with -q, -l, -c.
362
377
  In SYS1GREP_OPTS it is the default for every search, so every match goes to TOOL's provider too.
363
378
  With --dedup, each template's representative goes once, marked (×N like it). Over 200 KB nothing
364
379
  is sent to TOOL (exit 2): narrow the expression or add --dedup
380
+ --summarize=ask: the lines print as usual, then "Summarize these matches? [y/N]" on the terminal;
381
+ y pipes those lines to the TOOL of a bare --summarize, with no second search. Asked only when stdout
382
+ is a terminal too: not in a script, | less or $(...). -y summarizes unasked, also from SYS1GREP_OPTS;
383
+ -q and --dry-run never ask. With --dedup every line prints and TOOL gets the representatives.
384
+ The exit status stays the search's
365
385
  --no-summarize turn off an earlier --summarize (from SYS1GREP_OPTS, say): the lines print again
366
386
  --summarize-prompt=TEXT the user's own instruction, added after the fixed one in --summarize's system
367
387
  prompt (how long, what to focus on, ...). Needs --summarize; empty TEXT is the same as none
368
388
  --format=FORMAT plain (default) / markdown / html. With --rank, sys1grep writes the results itself: markdown a
369
- ## heading and a fenced block per result, html one document from --template, escaped,
370
- uncolored. With --summarize it is asked of TOOL instead (plain: no Markdown), and its
389
+ ## heading and a fenced block per result, html one document from --template, escaped, the
390
+ matches in <mark> (regex matches, else matching sentences, else the whole matching line; --color=never: none). With --summarize it is asked of TOOL instead (plain: no Markdown), and its
371
391
  answer prints as it comes, unchecked; html: TOOL writes plain text, which goes escaped into
372
392
  --template's {{answer}} and prints when TOOL is done. Before --summarize-prompt's TEXT, which can override it. Needs
373
393
  --rank (not with -l) or --summarize, except in SYS1GREP_OPTS. (It replaces --summarize-format.)
394
+ --serve[=PORT] serve a search page on 127.0.0.1 (a random free port, printed, or PORT) instead of searching: a box for
395
+ the meanings (-e -P -a -v -Q --step-to / --step-reverse-to), rank / summarize and the other
396
+ options (including -A/-B/-j/--max-cost, and -o with a sentence unit) as controls, results as
397
+ cards (--rank --format=html), and the command line for the current controls, to copy. Each
398
+ search runs this program with the options given here plus what the controls changed; the
399
+ targets and --chunk, -M, --max-filesize, -y, --edges, --template and the API settings stay as
400
+ given. With no target (and no -r, -g or git sys1grep) each search reads the bundled sample
401
+ corpus (examples/); a folder target without -r is refused at launch. The meanings come from
402
+ the page, so -e -P -a -v -Q --step-to --step-reverse-to and what the page replaces (-l -c -q -o
403
+ -z --unit=zero -H -i --format --color --dry-run --verbose) are refused at launch (-o is still a
404
+ Details control, just not a launch option).
405
+ A search over the cost guard (no -y, or --max-cost too low) is refused at once, naming -y and
406
+ --max-cost; the page shows the same estimate --dry-run would and a button to rerun that one search
407
+ with --max-cost just above it.
408
+ --suggest=Q:TEXT or --suggest=e:TEXT (or --suggest TEXT; repeat; a bare TEXT is a question) sets suggested-question buttons
374
409
  --template=NAME the document --rank or --summarize --format=html writes (default: SYS1GREP_TEMPLATE, else default):
375
410
  ~/.config/sys1grep/templates/NAME.html, else the bundled one (default, print, search, terminal); a
376
411
  value with / or ending in .html is a file. Placeholders: {{title}} {{query}} {{count}}, and
@@ -382,13 +417,18 @@ As git sys1grep, FILE arguments are pathspecs and every tracked file is searched
382
417
  file is kept. It and --template=list must stand alone, and are refused in SYS1GREP_OPTS
383
418
  --sys1-model=ID, --sys1-url=URL, --sys1-api-key=KEY
384
419
  the API settings, overriding SYS1GREP_MODEL, SYS1GREP_URL, SYS1GREP_API_KEY below.
385
- A key on the command line shows up in ps and shell history; prefer ~/.config/sys1grep/.env
386
- -h, --help this help (Japanese when LANG / LC_ALL / LC_MESSAGES starts with ja)
420
+ A key on the command line shows up in ps and shell history; prefer settings.json (below)
421
+ --lang=en|ja this help's language, and --serve's page default; overrides SYS1GREP_LANG below
422
+ -h, --help this help (--lang / SYS1GREP_LANG when set, else Japanese when the first set of LC_ALL /
423
+ LC_MESSAGES / LANG starts with ja)
387
424
  -V, --version print the version and exit
388
425
 
389
- Exit status: 0 matched / 1 no match / 2 error
426
+ Exit status: 0 matched / 1 no match / 2 error (bad arguments, unreadable file, API failure)
427
+ On 1, when lines were sent, stderr names the highest probability and its line (not with -q,
428
+ --step-to, --step-reverse-to, or when only a negation failed)
390
429
 
391
- Environment (read from the environment, else from ~/.config/sys1grep/.env; ./.env is never read):
430
+ Environment (read from the environment, else from ~/.config/sys1grep/settings.json, else from
431
+ ~/.config/sys1grep/.env; ./.env is never read):
392
432
  SYS1GREP_API_KEY API key. Falls back to TYPESAFE_API_KEY. Get one at https://console.typesafe.ai/
393
433
  SYS1GREP_URL endpoint (default https://api.typesafe.ai/v1/systemone). Any TypeSafe-compatible
394
434
  /v1/systemone works, e.g. https://openrouter.ai/api/v1/systemone
@@ -398,34 +438,47 @@ Environment (read from the environment, else from ~/.config/sys1grep/.env; ./.en
398
438
  ollama, lmstudio and a URL
399
439
  SYS1GREP_SUMMARIZER_API_KEY sent as Authorization: Bearer to a --summarize=URL server only (never
400
440
  SYS1GREP_API_KEY, which is Jev's)
401
- SYS1GREP_TEMPLATE --template's default, for --rank or --summarize --format=html
441
+ SYS1GREP_TEMPLATE --template's default, for --rank or --summarize --format=html; the environment or
442
+ .env only, not settings.json
443
+ SYS1GREP_CACHE --cache's default: on, off or only (default off); the environment or .env only, not
444
+ settings.json
445
+ SYS1GREP_LANG en or ja (default en): --help's language, and --serve's page default
446
+ OLLAMA_HOST the host:port --summarize=ollama talks to (default 127.0.0.1:11434), as for ollama
402
447
  SYS1GREP_OPTS default options, split on spaces and put before the command line, which wins;
403
448
  --no-X turns a boolean flag off (--color takes --color=never). Options only: no
404
449
  meanings, files or --. e.g. SYS1GREP_OPTS='--level strict -n'. Scripts: SYS1GREP_OPTS= sys1grep
405
450
  The key goes to SYS1GREP_URL, whatever it is. With SYS1GREP_URL set and no key, no auth header is sent.
406
- e.g. mkdir -p ~/.config/sys1grep && echo 'SYS1GREP_API_KEY=your-key' > ~/.config/sys1grep/.env`;
407
- const HELP_JA = `usage: sys1grep [OPTION]... -e MEANING|-Q QUESTION [-a MEANING] [-v MEANING]... [FILE...]
408
- sys1grep [OPTION]... EXPRESSION --step-to EXPRESSION [FILE...]
451
+ ~/.config/sys1grep/settings.json: a JSON object, every field optional, each the default for one variable:
452
+ url key model opts summarizer summarizerModel summarizerKey lang = SYS1GREP_URL SYS1GREP_API_KEY SYS1GREP_MODEL
453
+ SYS1GREP_OPTS SYS1GREP_SUMMARIZER SYS1GREP_SUMMARIZER_MODEL SYS1GREP_SUMMARIZER_API_KEY SYS1GREP_LANG. opts is
454
+ an array, one argument each. A variable set to empty counts as unset and falls through, except SYS1GREP_OPTS=,
455
+ which still drops opts. Keep the file at 0600.
456
+ e.g. {"key": "your-key", "opts": ["--level", "strict", "-n"]}`;
457
+ const HELP_JA = `usage: sys1grep [OPTION]... -e MEANING|-Q QUESTION|-P /RE/ [-a MEANING] [-v MEANING]... [FILE...]
458
+ sys1grep [OPTION]... -e START1 [-e START2]... --step-to END1 [-e END2]... [FILE...]
409
459
  jev (TypeSafe System One) で意味的にマッチする行を探す grep。FILE 省略時は stdin。
410
460
  git sys1grep として呼ぶと git grep と同じく FILE は pathspec になり、追跡中のファイルを全部探す。
411
461
 
412
- -e MEANING この意味に合う行 (複数指定は OR)
462
+ -e MEANING この意味に合う行 (複数指定は OR)。常に意味として扱う: /RE/FLAGS の形の値は
463
+ 拒否し、-P を案内する (例: -e '/catch/' is no longer a regex: use -P '/catch/')。
464
+ 本当に / で始まり / で終わる意味は、先頭にスペースを置けば通る (-e ' /cat/')
413
465
  -Q, --question QUESTION QUESTION に答えている行 (尋ねている行ではない)。-e "the line answers: QUESTION"
414
466
  と同じ。「ジョブはなぜ失敗したか」は「ディスクが満杯だった」に一致し、「サーバが落ちているか」は
415
- 否定の答えにも一致する。-e / -a / -v / ! とは -e と同じように組み合わせられる
416
- -a MEANING 直前の -e/-Q 項に AND で連結。-e A -a B -e C は (A and B) or C
417
- -v MEANING 直前の -e/-Q 項に AND NOT で連結。-e A -v B は A and not B
418
- 先頭に置けば単独の否定。-v B は not B (grep -v 相当)
419
- !MEANING -e / -Q / -a / -v のどこでも、先頭に ! を付けるとその意味だけ否定
420
- -e A -e '!B' は A or not B。-a '!C' は -v C と同じ
421
- /RE/FLAGS 正規表現項 (先頭と末尾が /、フラグは JS の dgimsuvy)。ローカルで判定しリクエストなし。
422
- 同じ AND 項の絞り込みになり、これが当たった行だけその項の意味を尋ねる
423
- -e '/ERROR|FATAL/' -a '/timeout/i' -v '/^DEBUG/' !/RE/ は上と同じく否定
467
+ 否定の答えにも一致する。-e / -a / -v / ! とは -e と同じように組み合わせられる。常に質問で、
468
+ 正規表現にはならない
469
+ -P /RE/FLAGS 正規表現項 (先頭と末尾が /、フラグは JS の dgimsuvy)。-e と同じく OR される。ローカルで
470
+ 判定しリクエストなし。同じ AND 項の絞り込みになり、これが当たったユニットだけその項の意味を尋ねる
471
+ -P '/ERROR|FATAL/' -a '/timeout/i' -v '/^DEBUG/'
424
472
  名前付き・番号付きグループは同じ項の意味に $<name>, $1-$99, $&, $$ として渡る
425
473
  (ECMAScript の置換パターン)。$<name> が存在しないグループを指す、または否定した
426
474
  正規表現のグループを指すのはエラー。$<name> と単一引用符を推奨 ($1 は sh/bash/zsh
427
- のダブルクォート内で生き残らない)。/ で始まり / で終わる本物の意味は、正規表現と
428
- 誤認されないよう先頭にスペースを置く
475
+ のダブルクォート内で生き残らない)。-P '!/RE/' は使えない。否定は -v '/RE/' で行う
476
+ -a MEANING 直前の -e/-Q/-P 項に AND で連結。-e A -a B -e C は (A and B) or C
477
+ -v MEANING 直前の -e/-Q/-P 項に AND NOT で連結。-e A -v B は A and not B
478
+ 先頭に置けば単独の否定。-v B は not B (grep -v 相当)。-a と -v も /RE/ を取れる
479
+ (正規表現の AND / AND NOT: -P '/RE/' -a '/one/' -v '/other/')
480
+ !MEANING -e / -Q / -a / -v のどこでも、先頭に ! を付けるとその意味だけ否定 (-P では不可)
481
+ -e A -e '!B' は A or not B。-a '!C' は -v C と同じ
429
482
  --level=LEVEL 厳しさ。肯定と否定の閾値をまとめて決める (既定 normal)
430
483
  loose : -t 0.3 -T 0.7 多少あやしくても拾う
431
484
  normal : -t 0.5 -T 0.5
@@ -436,10 +489,11 @@ git sys1grep として呼ぶと git grep と同じく FILE は pathspec にな
436
489
  -r ディレクトリを再帰的に探す (FILE 省略時はカレント)。.git、node_modules、
437
490
  .ssh/.aws/.gnupg/.kube/.docker、バイナリ、秘密情報らしいファイル (.env*, .netrc, .npmrc,
438
491
  .git-credentials, *.pem, *.key, id_rsa*...)、生成されたファイル (*.map, *.min.js,
439
- *.min.css, package-lock.json などのロックファイル) は飛ばす。git リポジトリの中では
492
+ *.min.css, package-lock.json などのロックファイル)、シンボリックリンク、settings.json 自身は飛ばす。
493
+ git リポジトリの中では
440
494
  git が無視するもの (.gitignore) も飛ばす。コマンドラインで指定したファイル・ディレクトリは
441
495
  それでも探す。.gz (ローテートしたログ) は zgrep のように展開して読み、その名前で表示する。
442
- 検索した行はすべて TypeSafe の API に送られる
496
+ 検索した行はすべて SYS1GREP_URL の先に送られる
443
497
  --cached git sys1grep 限定。作業ツリーではなくインデックス (ステージ済み) の blob を探す
444
498
  (git grep --cached と同じ)。作業ツリーから消したファイルもステージ済みなら見つかる。
445
499
  --untracked や <tree> とは併用できない
@@ -459,7 +513,8 @@ git sys1grep として呼ぶと git grep と同じく FILE は pathspec にな
459
513
  * は先頭のドットにも合う (*.md は .notes.md にも合う。rg --glob と同じ)
460
514
  --changed-within=WHEN -r と git sys1grep で、WHEN 以内に更新したファイルだけを探す。30m / 2h / 7d / 2w、
461
515
  日付か日時以降 (2026-09-01 はその日のローカル時刻 0 時、2026-09-01T09:00、...Z)、
462
- または today / this-week (月曜から) / this-month (ローカル時刻)。git の履歴ではなく mtime で見る
516
+ または today / this-week (月曜から) / this-month (ローカル時刻)。git の履歴ではなく mtime で見るので、
517
+ --cached や <tree> とは併用できない
463
518
  --no-auto-scope 意味の文面からファイルを絞り込まない。既定では意味ごとにまず Jev へ小さなリクエストを 1 つ
464
519
  送り、その意味が一致を言語・形式 (Python のファイル、YAML のファイル…) や変更時期 (1 分以内、
465
520
  1 時間以内、今日、昨日、1・2・3・7・30 日以内、今月、先週、先月、この 1 年、今年度) に限定して
@@ -474,11 +529,12 @@ git sys1grep として呼ぶと git grep と同じく FILE は pathspec にな
474
529
  絞らない。git sys1grep の pathspec は他と同じく絞る (--include と同じ)
475
530
  --auto-scope SYS1GREP_OPTS の --no-auto-scope を打ち消して、絞り込みを有効に戻す
476
531
  -g, --gitlog ファイルではなく git log のコミットを、1 コミット 1 レコード (ハッシュ、日付、件名、本文) で探す。
477
- 項が 1 つなら絞り込みを git log の引数にする: 時期は --since (昨日・先週・先月のように終わりが
478
- ある時期は --until も)、言語は pathspec、作者と自分は
479
- --author、このブランチと未プッシュは範囲。そのコマンドを stderr に出す。FILE は pathspec。
532
+ 項が 1 つなら絞り込みを git log の引数にする: 時期は --since-as-filter (2.37 より前の git では
533
+ --since。最初の古いコミットで止まる。昨日・先週・先月のように終わりがある時期は --until も)、
534
+ 言語は pathspec、作者と自分は --author、このブランチと未プッシュは範囲。そのコマンドを
535
+ stderr に出す。FILE は pathspec。
480
536
  -r と git sys1grep とは併用できない
481
- sys1grep -g -Q '今日、.mjsにおこなった性能向上の修正'
537
+ sys1grep -g -e '今日、.mjs に行った性能向上の修正'
482
538
  -l 一致した行ではなくファイル名だけを表示
483
539
  -H, --with-filename 1 ファイルだけでも、各行 (と -c の件数) の前にファイル名を付ける
484
540
  --no-filename 複数ファイル・-r・git sys1grep でもファイル名を付けない
@@ -487,10 +543,11 @@ git sys1grep として呼ぶと git grep と同じく FILE は pathspec にな
487
543
  -C NUM 前後 NUM 行を表示 (-A NUM -B NUM)
488
544
  -c 一致した行数だけをファイルごとに表示 (grep -c 相当)
489
545
  -q, --quiet 何も表示せず、最初の一致で止まる。一致があればエラーがあっても終了コード 0 (grep -q 相当)
490
- --chunk=LINES 1 リクエストにまとめる行数 (既定 30)
546
+ --chunk=LINES 1 リクエストにまとめる行数 (既定 30。質問も 1 リクエストに 64 個までなので、意味が 3 つ以上
547
+ なら行数はそれより少なくなる)
491
548
  同じリクエストの行は互いの文脈になるので、小さくすると速さだけでなく曖昧な行の
492
549
  判定も変わる
493
- -j N 同時リクエスト数 (既定 8)
550
+ -j N 同時リクエスト数 (既定 8。ローカル URL (localhost・127.0.0.1・[::1]) では 1)
494
551
  -n 行番号を付ける
495
552
  --unit=UNIT 判定の単位: line (既定)、zero (NUL 終端レコード)、sentence-by-jev / sentence-by-rule、function
496
553
  -z / --null-data は --unit=zero と同じ。sentence-by-* と併用すると、レコードごとに文に分ける
@@ -499,7 +556,7 @@ git sys1grep として呼ぶと git grep と同じく FILE は pathspec にな
499
556
  (grep -z と同じ)。ファイル名と件数は改行のまま。-n はレコード番号、-A/-B/-C は前後の
500
557
  レコード数、--chunk はレコード数を数える。レコードを出すツールと繋がる: git log -z、find -print0、
501
558
  xargs -0。例: git log -z --format='%h %s %b' | sys1grep -z -e "..."
502
- sentence-by-jev: 行ではなく文ごとに判定。上の CJK 文字に接する句点の無い改行を Jev にも聞く。
559
+ sentence-by-jev: 行ではなく文ごとに判定。CJK 文字に接する句点の無い改行を Jev にも聞く。
503
560
  句点で終わらない 1 行 1 件のデータをつながないため。そうした改行がある 30 行ごとに
504
561
  リクエストが 1 つ増える。出力は当たった文がかかる元の行のままで、文の部分を太字の黄で強調
505
562
  sentence-by-rule: 文単位の判定。規則だけで決める。追加のリクエストなし
@@ -513,11 +570,14 @@ git sys1grep として呼ぶと git grep と同じく FILE は pathspec にな
513
570
  (git grep -W と同じ)。関数名の行は diff=<driver> 属性と diff.<driver>.xfuncname で決め、
514
571
  無ければ .js/.ts/.py は sys1grep の規則、それ以外は英字・_・$ で始まる行。-M の既定は 8000。
515
572
  -z・-g・-o とは併用できない
516
- --step-to EXPRESSION 多段階マッチング: --step-to の前の式で開始ユニットを探し、そこから辺を幅優先でたどり、
517
- たどり着いたユニットのうち後ろの式に当たるもの (終点) までのパスを出す。step はホップではなく
518
- 検索の段: --step-to の後の -e / -a / -v / -Q はすべて終点の式。--step-to 意味 や --step-to '/RE/' は
519
- --step-to -e 意味 と同じ。たどる道はスコアで決めない。--hops の範囲でたどり着いたユニットを 1 回の
520
- バッチで終点の式と照らす。開始と終点の式が正規表現だけなら何も送らない。
573
+ --step-to END1 [-e END2]... 多段階マッチング、-e START1 [-e START2]... --step-to END1 [-e END2]...: START に当たる
574
+ ユニット (既定では関数) を探し、そこから辺 (呼び出し、または --edges) を幅優先でたどり、たどり
575
+ 着いたユニットのうち END に当たるもの (終点) までのパスを出す。2 回の検索 (step) を、何段でも
576
+ よい呼び出し (hop) でつなぐ。両側はそれぞれ別の式で、片側に -e を複数並べると OR。step は
577
+ ホップではなく検索の段: --step-to の後の -e / -P / -a / -v / -Q はすべて終点の式。--step-to 意味
578
+ は --step-to -e 意味 と同じ。--step-to '/RE/' は --step-to -P '/RE/' と同じ。たどる道はスコアで
579
+ 決めない。--hops の範囲でたどり着いたユニットを 1 回のバッチで終点の式と照らす。
580
+ 開始と終点の式が正規表現だけなら何も送らない。
521
581
  --edges が無ければ単位は関数 (--unit=function) で、辺は呼び出し: 関数本体 (コメント・docstring・
522
582
  文字列は除く) の 名前( が、その名前の関数すべてにつながる。
523
583
  パスは木の形で出す: ホップ数、file:line、関数名。終点は :、途中のユニットは - で区切る。-p は
@@ -525,11 +585,20 @@ git sys1grep として呼ぶと git grep と同じく FILE は pathspec にな
525
585
  -z・-g・-o・-c・-l・-A/-B/-C・--rank・--summarize・--dedup とは併用できない。
526
586
  --max-cost は終点のリクエストの前にもう一度、開始で送った分を含めて確かめる。--dry-run は
527
587
  開始の式が当たりうるユニットすべてから歩く (上限の見積もり)
588
+ 例:
589
+ # --summarize が行をツールに渡す所と、そこから呼ぶ関数のうちツールの失敗を扱うもの
528
590
  sys1grep -e "--summarize が行をツールに渡している" --step-to "ツールの起動や応答が失敗したときの処理" sys1grep.mjs
591
+ # 正規表現だけなので何も送らない: main 自身 (hop 0) と、main から呼ばれる関数のうち raise を含むもの
592
+ sys1grep -P '/^ *def main/' --step-to -P '/raise /' *.py
593
+ # 逆向き: main から helper に届くか (--step-reverse-to は呼ばれる側から呼ぶ側へたどる)
594
+ sys1grep -P '/^ *def helper/' --step-reverse-to -P '/^ *def main/' *.py
595
+ # 開始からちょうど 1〜2 回の呼び出しで届く終点だけ
596
+ sys1grep -P '/^ *def main/' --hops=1..2 --step-to -P '/raise /' *.py
529
597
  --edges=FILE 呼び出しの代わりにたどる辺。1 行 1 本: FROM_FILE:LINE<TAB>FROM_NAME<TAB>TO_FILE:LINE<TAB>TO_NAME。
530
598
  行はそれを含むユニットを指す (--unit は zero 以外ならどれでもよい)。import、リンク、ログ ID など
531
599
  どんな関係でもよい。向きの無い関係は両向きに書く
532
- --reverse 辺を逆向きにたどる (呼ばれる側から呼ぶ側へ)
600
+ --step-reverse-to END1 [-e END2]... --step-to と同じだが、辺を逆向きに (呼ばれる側から呼ぶ側へ) たどる。
601
+ --step-to とは併用できない
533
602
  --hops=N|M..N|M.. 終点のホップ数: ちょうど N、M から N、M 以上 (既定 0..)。ユニットのホップ数は、どれかの
534
603
  開始ユニットからの最短距離。ホップ 0 は開始ユニットそのもの
535
604
  -o 当たった部分だけを 1 行ずつ出す。正規表現の一致をそれぞれ出す (grep -o と同じ。意味だけで
@@ -541,34 +610,46 @@ git sys1grep として呼ぶと git grep と同じく FILE は pathspec にな
541
610
  検索するファイル (単位の数と送る数) と各リクエストの質問を表示する (質問は文面ごとにまとめて
542
611
  数え、行の ID は Lnnn と表示)。キーの値は表示しない。--dedup と --unit=sentence-by-jev の事前の問い合わせは
543
612
  no と答えたものとして数えるので、その場合の数は目安。最後の行に
544
- 入力トークン数と、TypeSafe 本体なら料金の見積もりを出す (~ 付き、誤差 1 割程度)
613
+ 入力トークン数と料金の見積もりを出す (~ 付き、誤差 1 割程度。誰の価格かは --max-cost の項)
545
614
  --verbose 同じ表示を検索しながら stderr に出す。端末でなくても最後の集計行を出す
546
615
  -i, --interactive まず --dry-run と同じ内容 (ファイル・行数・リクエスト数) を見せて端末で聞き、
547
616
  y のときだけ検索する。答えるまで何も送らない。端末が無ければエラー
548
617
  -M NUM, --max-columns=NUM 行 (-z ならレコード) の先頭 NUM 文字までを送る。それでも検索・判定は
549
- される (NUM より先にある一致だけは見つからない)。既定 2000 (-z なら 8000)
618
+ される (NUM より先にある一致だけは見つからない)。既定 2000 (-z と --unit=function なら 8000)
550
619
  --max-filesize=SIZE 読む前に、検索対象を 1 つずつ計測する
551
620
  (K/M/G の接尾辞、既定 10M)。これを超えるものは rg の --max-filesize と同じく無条件に飛ばし、
552
621
  stderr に名前を出す。-y は効かない (明示的に指定したファイルでも超えていれば飛ばす)。
553
622
  標準入力は読んでから計測し、超えていれば同じく飛ばす。--cached / <tree>: の対象は blob
554
623
  で、その内容から計測する。.gz は展開後の大きさで計測する (上限で展開を止める)。
555
624
  -g のコミットはここに含まれない (送るときに -M ですでに上限がある)
556
- --max-cost=USD 送る予定の入力トークンを見積もって値段を出す。これ (既定 1) を超えたら端末で続けるか
625
+ --max-cost=USD 送る予定の入力トークンを見積もって値段を出す (ローカル URL は無料、jev-latest 0.042、clef 0.24、
626
+ clef-flash 0.09 USD/M トークン、他のモデルは Jev と同じ)。これ (既定 1) を超えたら端末で続けるか
557
627
  聞く。端末が無く上限を超えていれば終了コード 2 で、どのオプションを緩めれば通るかを言う。
558
628
  -q でも変わらない (スクリプトからは -y)。
559
629
  -i は無条件かつこれより前に聞くので、二重には聞かない
560
630
  -y, --yes その質問に yes と答えて聞かない (スクリプトからは SYS1GREP_OPTS='--max-cost ... -y')
631
+ --cache[=on|off|only] Jev の答えを ~/.config/sys1grep/cache に残す。キーは送る内容 (モデル・行・問い) と
632
+ 送信先のホスト名で、同じリクエストはそこから答え、送らず料金もかからない。既定は off
633
+ (SYS1GREP_CACHE): 検索した行が残るため。値の無い --cache は on。only はキャッシュだけで答え、
634
+ 無いリクエストがあればそれを名指しして終了コード 2 (何も送らない。-q が先に一致を見つけて
635
+ いれば 0)。期限は無い: Jev の答えは実行ごとに 0.05 ほど揺れ、残るのはその 1 回分。
636
+ --dry-run はキャッシュ済みと送る分の数を出す。
637
+ --cache=clear で空にする。これは単独で使い、SYS1GREP_OPTS には書けない
561
638
  --dedup[=auto|always|never] 全行ではなくテンプレートごとに 1 行だけ判定する。ID・ハッシュ・数値・日付と時刻・
562
639
  パス・URL だけが違う行は同じテンプレートとみなし、代表 1 行を送ってその答えを残りにも使う。
563
640
  どれをまとめてよいかは意味による。「ディスク使用率が 90% を超えている」なら数値が、
564
- 「夜間に起きた」なら時刻が答えを決める。そこで意味ごとに小さなリクエストを 1 つ送って
565
- Jev に聞き、答えを変えうる種類はまとめない。機械が吐くログ向けで、費用が 30 分の 1
566
- 以下になることもある。散文には共通の骨格がないのでほとんど縮まず、値を読む意味もあまり
641
+ 「夜間に起きた」なら時刻が答えを決める。always は意味ごとに小さなリクエストを 1 つ送って
642
+ Jev に聞き、答えを変えうる種類はまとめない。auto は聞かず、各テンプレートから最大 5 行の
643
+ 標本 (変わる値ごとに最小・最大・中央の行) を判定する。判定が揃えば、標本でない行は最初の
644
+ 標本の答えを共有し、標本は自分の答えを保つ。分かれたら --max-cost をもう一度確かめてから
645
+ 残りを送る (never より数パーセント高くつく)。標本のあいだにしか無い値は見落とす。
646
+ --dry-run はすべての標本が揃うと仮定するので、その数は下限。機械が吐くログ向けで、
647
+ 費用が 30 分の 1 以下になることもある。散文には共通の骨格がないのでほとんど縮まず、値を読む意味もあまり
567
648
  縮まない。
568
649
  never (既定。当面は #143 参照) は尋ねも縮めもしないが、送る前にローカルで、全種類を
569
650
  縮めた場合 (最良のケース) にどれだけ浮くかを見積もる。それが --dedup 自身の質問の費用の
570
651
  2 倍以上なら、stderr に --dedup=auto を勧める一行を出す (-q では出さない)。auto はその
571
- 見積もりが割に合うときだけ尋ねて縮める。裸の --dedup は --dedup=always で、この
652
+ 見積もりが割に合うときだけ縮める。裸の --dedup は --dedup=always で、この
572
653
  オプションが値を取る前の --dedup と同じく常に尋ねて縮める。--verbose・--dry-run は
573
654
  値ごとにこの判定と数字を表示する。--unit=zero や --unit=sentence-by-* ではレコードや文を
574
655
  単位にまとめる
@@ -586,8 +667,9 @@ git sys1grep として呼ぶと git grep と同じく FILE は pathspec にな
586
667
  --summarize[=TOOL] 出力するはずの内容 (ファイル名・-n・-A/-B/-C・-p) を TOOL に渡し、意味に照らした要約を
587
668
  頼んで、その答えを代わりに表示する。TOOL: claude (既定。SYS1GREP_SUMMARIZER で変えられる)。
588
669
  claude -p --model haiku をツールなし・設定なしで動かす。llm (Simon Willison 氏の、同じくツールなし)。
589
- pi (--no-tools と --no-session・--no-context-files・--no-extensions・--no-skills・
590
- --no-prompt-templates すべて)。ollama・lmstudio (ローカルの OpenAI 互換サーバ、CLI なし)。または http(s):// URL、任意の
670
+ pi (--no-tools・--no-approve・--thinking off と --no-session・--no-context-files・--no-extensions・
671
+ --no-skills・--no-prompt-templates すべて)。ollama・lmstudio (ローカルの OpenAI 互換サーバ、
672
+ CLI なし)。または http(s):// URL、任意の
591
673
  OpenAI 互換サーバへ同じリクエストを送る (llama.cpp・vLLM・LocalAI・ゲートウェイ)。
592
674
  ollama・lmstudio・URL は SYS1GREP_SUMMARIZER_MODEL が要る (既定モデルが無い)。
593
675
  一致した行は TOOL の提供元へもう一度送られる (ollama・lmstudio・ローカル URL ならどこへも送られない)。
@@ -595,15 +677,32 @@ git sys1grep として呼ぶと git grep と同じく FILE は pathspec にな
595
677
  SYS1GREP_OPTS に書くとすべての検索の既定になり、一致した行は毎回 TOOL の提供元へも送られる。
596
678
  --dedup ではテンプレートごとに代表を 1 回だけ、(×N like it) を付けて渡す。200 KB を超えたら
597
679
  TOOL には何も渡さない (終了コード 2)。式を絞るか --dedup を付ける
680
+ --summarize=ask: 行はいつもどおり出し、続けて端末で「Summarize these matches? [y/N]」と聞く。
681
+ y なら出した行を値なしの --summarize の TOOL に渡す (検索はやり直さない)。標準出力も端末のときだけ
682
+ 聞く (スクリプト・| less・$(...) では聞かない)。-y は聞かずに要約する (SYS1GREP_OPTS の -y でも)。
683
+ -q と --dry-run は聞かない。--dedup では行はすべて出し、TOOL には代表を渡す。終了コードは検索のもの
598
684
  --no-summarize それより前の --summarize (SYS1GREP_OPTS のものなど) を取り消し、行をそのまま出す
599
685
  --summarize-prompt=TEXT --summarize のシステムプロンプトに、固定の指示に続けて足すユーザー自身の指示
600
686
  (長さ・観点など)。--summarize が要る。空の TEXT は指定しないのと同じ
601
687
  --format=FORMAT plain (既定) / markdown / html。--rank では sys1grep 自身が結果を書く。markdown は結果ごとに
602
- ## 見出しとコードブロック、html は --template の 1 つの文書で、文字はエスケープし色は付けない。
688
+ ## 見出しとコードブロック、html は --template の 1 つの文書で、文字はエスケープし、一致は <mark> で示す (正規表現の一致、無ければ一致した文、無ければ一致した行全体。--color=never なら示さない)。
603
689
  --summarize では代わりに TOOL に頼み (plain は Markdown なし)、答えは確かめずにそのまま表示する。
604
690
  html は TOOL に平文を書かせ、エスケープして --template の {{answer}} に入れ、TOOL が終わってから表示する。
605
691
  --summarize-prompt の TEXT より前に置くので TEXT で上書きできる。
606
692
  --rank (-l とは併用不可) か --summarize が要る (SYS1GREP_OPTS では要らない)。--summarize-format の後継
693
+ --serve[=PORT] 検索せずに、127.0.0.1 で検索ページを出す (空いているポートを選んで表示、または PORT)。意味を
694
+ 入れる欄 (-e -P -a -v -Q --step-to / --step-reverse-to)、rank / summarize などのオプション (-A/-B/-j/--max-cost、文単位
695
+ のときだけの -o も含む) をコントロールにし、結果はカード (--rank --format=html)、いまのコントロール
696
+ のコマンドラインをコピー用に表示する。検索のたびにここで指定したオプションにコントロールの変更を
697
+ 足してこのプログラムを実行する。対象と --chunk・-M・--max-filesize・-y・--edges・--template・API 設定は
698
+ 指定のまま。対象が無ければ (-r・-g・git sys1grep でもなければ) 同梱のサンプル (examples/) を探す。
699
+ -r の無いフォルダの対象は起動時に拒む。意味はページから来るので -e -P -a -v -Q --step-to
700
+ --step-reverse-to と、起動時にページが置き換えるもの (-l -c -q -o -z --unit=zero -H -i --format
701
+ --color --dry-run --verbose) は受け付けない (-o は起動オプションではないが、
702
+ コントロールとしては使える)。費用の確認を越える検索はその場で拒まれ、-y と --max-cost を名指しする。
703
+ ページはその拒否が持つ見積もり (--dry-run と同じもの) を示し、--max-cost をそれより上げてその検索
704
+ だけやり直すボタンを出す。
705
+ --suggest=Q:TEXT か --suggest=e:TEXT (--suggest TEXT でもよい。繰り返せる。TEXT だけなら質問) でおすすめ質問のボタンを指定する
607
706
  --template=NAME --rank か --summarize の --format=html が書く文書 (既定は SYS1GREP_TEMPLATE、無ければ default)。
608
707
  ~/.config/sys1grep/templates/NAME.html、無ければ同梱のもの (default, print, search, terminal)。/ を含むか
609
708
  .html で終わる値はファイル。置き換える文字列は {{title}} {{query}} {{count}} と、<!--result--> と
@@ -616,13 +715,18 @@ git sys1grep として呼ぶと git grep と同じく FILE は pathspec にな
616
715
  SYS1GREP_OPTS には書けない
617
716
  --sys1-model=ID, --sys1-url=URL, --sys1-api-key=KEY
618
717
  API の設定。下の SYS1GREP_MODEL / SYS1GREP_URL / SYS1GREP_API_KEY より優先。
619
- コマンドラインのキーは ps やシェル履歴に残るので、なるべく ~/.config/sys1grep/.env に書く
620
- -h, --help このヘルプ (LANG / LC_ALL / LC_MESSAGES が ja 以外なら英語)
718
+ コマンドラインのキーは ps やシェル履歴に残るので、なるべく settings.json (下) に書く
719
+ --lang=en|ja このヘルプの言語、および --serve のページの既定言語。下の SYS1GREP_LANG より優先
720
+ -h, --help このヘルプ (--lang / SYS1GREP_LANG が設定されていればそれ、無ければ LC_ALL / LC_MESSAGES /
721
+ LANG のうち最初に設定されたものが ja 以外なら英語)
621
722
  -V, --version バージョンを表示して終了
622
723
 
623
724
  終了コード: 一致あり 0 / なし 1 / エラー 2 (引数・読めないファイル・API 障害)
725
+ 行を送って 1 のときは、最も高かった確率とその行を stderr に出す (-q・--step-to・--step-reverse-to と、
726
+ 否定だけで落ちたときは出さない)
624
727
 
625
- 環境変数 (環境、無ければ ~/.config/sys1grep/.env から読む。./.env は読まない):
728
+ 環境変数 (環境、無ければ ~/.config/sys1grep/settings.json、それも無ければ ~/.config/sys1grep/.env から読む。
729
+ ./.env は読まない):
626
730
  SYS1GREP_API_KEY API キー。無ければ TYPESAFE_API_KEY。取得は https://console.typesafe.ai/
627
731
  SYS1GREP_URL 送信先 (既定 https://api.typesafe.ai/v1/systemone)。TypeSafe 互換の
628
732
  /v1/systemone なら可。例 https://openrouter.ai/api/v1/systemone
@@ -632,20 +736,30 @@ git sys1grep として呼ぶと git grep と同じく FILE は pathspec にな
632
736
  ollama・lmstudio・URL では必須
633
737
  SYS1GREP_SUMMARIZER_API_KEY --summarize=URL のサーバへ Authorization: Bearer で送る
634
738
  (Jev 用の SYS1GREP_API_KEY とは別)
635
- SYS1GREP_TEMPLATE --rank か --summarize の --format=html の --template の既定
739
+ SYS1GREP_TEMPLATE --rank か --summarize の --format=html の --template の既定。環境か .env だけで、
740
+ settings.json には書けない
741
+ SYS1GREP_CACHE --cache の既定: on・off・only (既定 off)。環境か .env だけで、settings.json には書けない
742
+ SYS1GREP_LANG en または ja (既定 en)。--help の言語、および --serve のページの既定言語
743
+ OLLAMA_HOST --summarize=ollama の接続先 host:port (既定 127.0.0.1:11434)。ollama と同じ
636
744
  SYS1GREP_OPTS 既定のオプション。空白で区切ってコマンドラインの前に置くので、コマンドラインが
637
745
  優先する。--no-X で真偽のフラグを消せる (--color は --color=never)。書けるのは
638
746
  オプションだけで、意味・ファイル・-- は書けない。例 SYS1GREP_OPTS='--level strict -n'。
639
747
  スクリプトからは SYS1GREP_OPTS= sys1grep と空にして呼ぶ
640
748
  キーは SYS1GREP_URL の先へそのまま送られる。SYS1GREP_URL 指定時にキーが無ければ認証ヘッダを付けない。
641
- 例: mkdir -p ~/.config/sys1grep && echo 'SYS1GREP_API_KEY=your-key' > ~/.config/sys1grep/.env`;
749
+ ~/.config/sys1grep/settings.json: JSON のオブジェクト。どの項目も省略でき、それぞれ 1 つの環境変数の既定値になる:
750
+ url key model opts summarizer summarizerModel summarizerKey lang = SYS1GREP_URL SYS1GREP_API_KEY SYS1GREP_MODEL
751
+ SYS1GREP_OPTS SYS1GREP_SUMMARIZER SYS1GREP_SUMMARIZER_MODEL SYS1GREP_SUMMARIZER_API_KEY SYS1GREP_LANG。opts は配列で、
752
+ 1 要素が 1 引数。環境変数は空だと未設定扱いでこの先へ進むが、SYS1GREP_OPTS= だけは空のまま勝ち、opts を外す。
753
+ ファイルの権限は 0600 にしておく
754
+ 例: {"key": "your-key", "opts": ["--level", "strict", "-n"]}`;
642
755
  if (opt.version) {
643
756
  console.log(`sys1grep ${JSON.parse(readFileSync(new URL('package.json', import.meta.url), 'utf8')).version}`);
644
757
  process.exit(0);
645
758
  }
646
759
  if (opt.help) {
647
760
  const locale = process.env.LC_ALL || process.env.LC_MESSAGES || process.env.LANG || '';
648
- console.log(locale.startsWith('ja') ? HELP_JA : HELP_EN);
761
+ const helpLang = langValue ?? (locale.startsWith('ja') ? 'ja' : 'en');
762
+ console.log(helpLang === 'ja' ? HELP_JA : HELP_EN);
649
763
  process.exit(0);
650
764
  }
651
765
  // --rank --format=html's document. A template is an HTML file split by <!--result--> and <!--/result--> into the
@@ -671,6 +785,16 @@ if (opt['install-templates']) {
671
785
  }
672
786
  process.exit(0);
673
787
  }
788
+ // #252: Jev's answers, one file per request, named by the hash of the body sent and the endpoint's host. The
789
+ // questions carry the searched lines, so the cache is off unless asked for.
790
+ const CACHE_DIR = `${homedir()}/.config/sys1grep/cache`;
791
+ if (opt.cache === 'clear') {
792
+ if (tokens.filter(k => k.index >= defaults.length).length > 1) die('--cache=clear takes no other arguments');
793
+ rmSync(CACHE_DIR, { recursive: true, force: true });
794
+ process.exit(0);
795
+ }
796
+ const cacheMode = opt.cache ?? (process.env.SYS1GREP_CACHE || 'off');
797
+ if (!['on', 'off', 'only'].includes(cacheMode)) die(opt.cache === undefined ? `SYS1GREP_CACHE: '${cacheMode}' is not on, off or only` : `--cache: '${cacheMode}' is not on, off, only or clear`);
674
798
  const parseTemplate = (text, where) => {
675
799
  const open = '<!--result-->', close = '<!--/result-->', times = m => text.split(m).length - 1;
676
800
  for (const m of [open, close]) if (times(m) !== 1) throw new Error(`template ${where}: ${times(m) ? `${m} appears ${times(m)} times` : `no ${m}`}; it needs one ${open} ... ${close} around the part repeated per result`);
@@ -680,14 +804,28 @@ const parseTemplate = (text, where) => {
680
804
  return { head, item, tail };
681
805
  };
682
806
  const esc = t => t.replace(/[&<>"']/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' })[c]);
683
- const fillPart = (part, vars) => part.replace(/\{\{(\w+)\}\}/g, (m, k) => (Object.hasOwn(vars, k) ? esc(String(vars[k])) : m));
807
+ // {{lines}}: the \u0001 / \u0002 that markup() put around a match become <mark> (a source's own ones are stripped there)
808
+ const fillPart = (part, vars) => part.replace(/\{\{(\w+)\}\}/g, (m, k) => (Object.hasOwn(vars, k) ? (k === 'lines' ? esc(String(vars[k])).replace(/\u0001/g, '<mark>').replace(/\u0002/g, '</mark>') : esc(String(vars[k]))) : m));
684
809
 
685
810
  const customUrl = opt['sys1-url'] || SYS1GREP_URL;
686
811
  const apiUrl = customUrl || 'https://api.typesafe.ai/v1/systemone';
687
812
  const apiHost = (() => { try { return new URL(apiUrl).host; } catch { die(`not a URL: ${apiUrl} (--sys1-url / SYS1GREP_URL)`); } })();
688
813
  const model = opt['sys1-model'] || SYS1GREP_MODEL || 'jev-latest';
689
- const credential = opt['sys1-api-key'] || SYS1GREP_API_KEY || TYPESAFE_API_KEY;
690
- if (credential && new URL(apiUrl).protocol === 'http:' && !/^(localhost|127\.0\.0\.1|\[::1\])(:\d+)?$/.test(apiHost))
814
+ // The price shown and checked by --max-cost (#174), USD per M input tokens, unless the endpoint reports its own cost
815
+ // (usage.cost): a local URL is free, a model in PRICES has its own, anything else is Jev's list price. The host as typed
816
+ // decides locality, with no DNS lookup, and only for a URL the user gave (SYS1GREP_URL / --sys1-url).
817
+ const PRICES = { 'jev-latest': 0.042, clef: 0.24, 'clef-flash': 0.09 };
818
+ const LOCAL_HOST = /^(localhost|127\.0\.0\.1|\[::1\])(:\d+)?$/;
819
+ const localUrl = !!customUrl && LOCAL_HOST.test(apiHost);
820
+ // A local decision model serves one request at a time: Jeff answers a second with 529, strands-decider crashes (#180,
821
+ // #181). So a local URL defaults to -j 1; a -j given anywhere (command line, SYS1GREP_OPTS, settings.json) wins.
822
+ const localJ = localUrl && optSrc('j') === null;
823
+ if (localJ) opt.j = '1';
824
+ const pricePerM = localUrl ? 0 : Object.hasOwn(PRICES, model) ? PRICES[model] : PRICES['jev-latest'];
825
+ const priceTag = localUrl ? ' (local URL, free)' : !customUrl ? ''
826
+ : Object.hasOwn(PRICES, model) && model !== 'jev-latest' ? ` at ${model}'s list price` : " at TypeSafe's list price";
827
+ const credential = opt['sys1-api-key'] || envAPI_KEY.value;
828
+ if (credential && new URL(apiUrl).protocol === 'http:' && !LOCAL_HOST.test(apiHost))
691
829
  console.error(`sys1grep: warning: the API key goes to ${apiHost} over plain http`);
692
830
  // --dry-run prints the files and the requests that would be sent, to stdout, and sends nothing. --verbose prints
693
831
  // the same to stderr while searching. -q's early exits would cut the list short, so --dry-run turns -q off.
@@ -715,8 +853,7 @@ if (process.stderr.isTTY && process.env.TERM !== 'dumb' && !opt.quiet && !dry &&
715
853
 
716
854
  // Expression: a list of AND terms joined by OR. Each literal is a meaning { kind:'m', text, not } or a
717
855
  // regex { kind:'r', re, names, count, not }, matched locally. A leading ! negates just that literal.
718
- // /pattern/flags (first and last char /, JS flags) is a regex term; anything else is a meaning.
719
- const RE_SHAPE = /^\/(.*)\/([dgimsuvy]*)$/s;
856
+ // /pattern/flags (first and last char /, JS flags, RE_SHAPE above) is a regex term; anything else is a meaning.
720
857
  function compileRegex(pattern, flags) {
721
858
  let re, probe;
722
859
  try { re = new RegExp(pattern, flags); probe = new RegExp(`${pattern}|`, flags).exec(''); }
@@ -724,32 +861,42 @@ function compileRegex(pattern, flags) {
724
861
  return { re, names: new Set(Object.keys(probe.groups ?? {})), count: probe.length - 1 };
725
862
  }
726
863
  // Multi-step matching (#163): a term before --step-to is the start expression's, one after it the end's, and
727
- // --step-to starts a new expression, so -a or -v right after it never joins a start term.
864
+ // --step-to starts a new expression, so -a or -v right after it never joins a start term. --step-reverse-to is
865
+ // --step-to walking the edges backwards.
728
866
  const expr = [];
729
- const multiStep = tokens.some(tk => tk.kind === 'option' && tk.name === 'step-to');
867
+ const steps = tokens.filter(tk => tk.kind === 'option' && ['step-to', 'step-reverse-to'].includes(tk.name));
868
+ const multiStep = steps.length > 0, reverse = steps.some(tk => tk.name === 'step-reverse-to');
869
+ const stepFlag = reverse ? '--step-reverse-to' : '--step-to';
870
+ if (new Set(steps.map(tk => tk.name)).size > 1) die('--step-to and --step-reverse-to cannot be combined: a step walks one way');
730
871
  let role = multiStep ? 'start' : undefined;
731
872
  for (const tk of tokens) {
732
- if (tk.kind === 'option' && tk.name === 'step-to') {
733
- if (role === 'end') die('--step-to cannot be given twice: only one step is supported');
873
+ if (steps.includes(tk)) {
874
+ if (role === 'end') die(`${stepFlag} cannot be given twice: only one step is supported`);
734
875
  role = 'end';
735
876
  continue;
736
877
  }
737
- if (tk.kind !== 'option' || !['e', 'a', 'v', 'question'].includes(tk.name)) continue;
878
+ if (tk.kind !== 'option' || !['e', 'P', 'a', 'v', 'question'].includes(tk.name)) continue;
738
879
  const fresh = expr.at(-1)?.role !== role;
739
- if (tk.name === 'a' && (expr.length === 0 || fresh)) die(`-a needs a preceding -e${role === 'end' ? ' after --step-to' : ''}`);
740
- const bang = tk.value.startsWith('!'); // per-literal negation: "!MEANING" / "!/re/"
880
+ if (tk.name === 'a' && (expr.length === 0 || fresh)) die(`-a needs a preceding -e/-P${role === 'end' ? ` after ${stepFlag}` : ''}`);
881
+ // -P is always a regex term, never negated itself (a regex NOT is -v '/RE/').
882
+ if (tk.name === 'P' && tk.value.startsWith('!')) die(`-P '${tk.value}' is not allowed: negate a regex with -v '/RE/'`);
883
+ const bang = tk.name !== 'P' && tk.value.startsWith('!'); // per-literal negation: "!MEANING" / "!/re/"
741
884
  const bare = bang ? tk.value.slice(1) : tk.value;
742
885
  if (!bare.trim()) die(`${tk.name.length > 1 ? '--' : '-'}${tk.name}: MEANING must not be empty`);
743
886
  const not = bang !== (tk.name === 'v');
744
- const shape = tk.name !== 'question' && RE_SHAPE.exec(bare); // -Q is always a question, never a regex
887
+ const shape = tk.name !== 'question' && RE_SHAPE.exec(bare); // -Q is always a question, never a regex (unchanged by #238)
888
+ // -e is always a meaning now: a /RE/flags value there is refused, naming -P (the new regex option). A real
889
+ // meaning that looks like /RE/flags still gets through with a leading space (" /cat/"), which breaks the shape.
890
+ if (tk.name === 'e' && shape) die(`-e '${tk.value}' is no longer a regex: use ${bang ? `-v '${bare}'` : `-P '${bare}'`}; for the text itself, add a leading space`);
891
+ if (tk.name === 'P' && !shape) die(`-P '${tk.value}' must be a regex (/RE/flags)`);
745
892
  const lit = shape ? { kind: 'r', not, ...compileRegex(shape[1], shape[2]) }
746
- : { kind: 'm', not, text: tk.name === 'question' ? `the line answers: ${bare}` : bare };
747
- if (tk.name === 'e' || tk.name === 'question' || expr.length === 0 || fresh) expr.push(Object.assign([lit], { role }));
893
+ : { kind: 'm', not, text: tk.name === 'question' ? `the line answers: ${bare}` : bare, said: bare };
894
+ if (['e', 'P', 'question'].includes(tk.name) || expr.length === 0 || fresh) expr.push(Object.assign([lit], { role }));
748
895
  else expr.at(-1).push(lit);
749
896
  }
750
- if (multiStep && !expr.some(term => term.role === 'start')) die('--step-to needs an expression before it to start from: -e MEANING --step-to ...');
751
- if (multiStep && !expr.some(term => term.role === 'end')) die("--step-to needs an expression: --step-to MEANING, --step-to '/RE/', or --step-to -e ...");
752
- if (!expr.length) die('no -e MEANING or -Q QUESTION given');
897
+ if (multiStep && !expr.some(term => term.role === 'start')) die(`${stepFlag} needs an expression before it to start from: -e MEANING ${stepFlag} ...`);
898
+ if (multiStep && !expr.some(term => term.role === 'end')) die(`${stepFlag} needs an expression: ${stepFlag} MEANING, ${stepFlag} '/RE/', or ${stepFlag} -e ...`);
899
+ if (!expr.length) die("no -e MEANING, -Q QUESTION, or -P '/RE/' given");
753
900
  // What a run judges first: the whole expression, or with --step-to the start expression; the end expression
754
901
  // is judged later, on the units the walk reaches.
755
902
  const starting = multiStep ? expr.filter(term => term.role === 'start') : expr;
@@ -785,7 +932,7 @@ if (opt.rank !== undefined) {
785
932
  // A compatible local server may need no key; the TypeSafe default always does. Regex-only queries never call the API.
786
933
  // --unit=sentence-by-jev asks Jev where wrapped lines join; with regex terms only, nothing else is sent, so rules decide.
787
934
  if (opt.unit === 'sentence-by-jev' && !hasMeanings) opt.unit = 'sentence-by-rule';
788
- if (hasMeanings && !credential && !customUrl) die('SYS1GREP_API_KEY is not set. Export it or put it in ~/.config/sys1grep/.env');
935
+ if (hasMeanings && !credential && !customUrl) die(`SYS1GREP_API_KEY is not set. Export it or put it in ${SETTINGS_SHOWN} as "key"`);
789
936
 
790
937
  const levels = { loose: [0.3, 0.7], normal: [0.5, 0.5], strict: [0.7, 0.3] };
791
938
  const level = Object.hasOwn(levels, opt.level) && levels[opt.level];
@@ -793,6 +940,9 @@ if (!level) die(`--level must be one of ${Object.keys(levels).join(', ')}`);
793
940
  const tPos = opt.t === undefined ? level[0] : Number(opt.t);
794
941
  const tNeg = opt.T === undefined ? level[1] : Number(opt.T);
795
942
  const chunkLines = Number(opt.chunk);
943
+ // Clef's input schema caps a request at 64 questions (#174); every backend gets that cap, which only adds a few requests
944
+ // when a line carries 3 or more meanings, or auto-scope has many candidates.
945
+ const MAX_QUESTIONS = 64;
796
946
  // Validate numeric options. parseArgs turns -C=10 into the value "=10", so reject that here.
797
947
  for (const [k, label] of [['t', '-t'], ['T', '-T'], ['chunk', '--chunk'], ['j', '-j'], ['A', '-A'], ['B', '-B'], ['C', '-C'], ['max-columns', '-M'], ['max-cost', '--max-cost']])
798
948
  if (opt[k] !== undefined && !(k === 't' || k === 'T' || k === 'max-cost' ? /^\d+(\.\d+)?$/ : /^\d+$/).test(opt[k])) die(`${label}: invalid number '${opt[k]}' (write ${label} 10 or ${label}10, not ${label}=10)`);
@@ -811,6 +961,10 @@ const parseSize = (s, label) => {
811
961
  const fmtSize = b => (b >= 1024 ** 3 ? `${(b / 1024 ** 3).toFixed(1)} GB` : b >= 1024 ** 2 ? `${(b / 1024 ** 2).toFixed(1)} MB` : `${Math.ceil(b / 1024)} KB`);
812
962
  const MAX_FILESIZE = parseSize(opt['max-filesize'] ?? '10M', '--max-filesize');
813
963
  const MAX_COST = Number(opt['max-cost']);
964
+ // #138: -r / git sys1grep warn above this many units sent through a term with no regex. #132's git-log run sent
965
+ // ~1,200 lines and the runaway matplotlib one ~215,000 (django's 7.4M tokens, ~35,000): 10,000 sits between them,
966
+ // ~2M tokens (~$0.08), well under --max-cost's default $1 (~120,000 lines).
967
+ const LARGE_SEND_UNITS = 10_000;
814
968
  if (!['line', 'zero', 'sentence-by-jev', 'sentence-by-rule', 'function'].includes(opt.unit)) die('--unit must be line, zero, sentence-by-jev, sentence-by-rule, or function');
815
969
  if (!['auto', 'always', 'never'].includes(opt.color)) die('--color must be auto, always or never');
816
970
  if (!['auto', 'always', 'never'].includes(opt.dedup)) die('--dedup must be auto, always or never');
@@ -829,11 +983,11 @@ let edgeList = null; // [{ at: [file, line], to: [file, line], name, toName }] f
829
983
  if (multiStep) {
830
984
  const off = [['z', opt.gitlog ? '-g' : '-z'], ['o', '-o'], ['c', '-c'], ['l', '-l'], ['A', '-A'], ['B', '-B'], ['C', '-C'], ['rank', '--rank'], ['summarize', '--summarize']]
831
985
  .find(([k]) => opt[k] !== undefined && opt[k] !== false);
832
- if (off) die(`--step-to cannot be combined with ${off[1]}: it prints paths, not lines`);
833
- if (opt.dedup !== 'never' && optSrc('dedup') === '') die('--step-to cannot be combined with --dedup: each unit is judged as itself');
986
+ if (off) die(`${stepFlag} cannot be combined with ${off[1]}: it prints paths, not lines`);
987
+ if (opt.dedup !== 'never' && optSrc('dedup') === '') die(`${stepFlag} cannot be combined with --dedup: each unit is judged as itself`);
834
988
  opt.dedup = 'never';
835
989
  if (opt.edges === undefined && opt.unit !== 'function') {
836
- if (optSrc('unit') === '') die(`--step-to walks calls between functions: --unit=${opt.unit} needs --edges`);
990
+ if (optSrc('unit') === '') die(`${stepFlag} walks calls between functions: --unit=${opt.unit} needs --edges`);
837
991
  opt.unit = 'function';
838
992
  }
839
993
  if (opt.edges !== undefined) {
@@ -846,7 +1000,7 @@ if (multiStep) {
846
1000
  return { at: place(at), to: place(to), name, toName };
847
1001
  });
848
1002
  }
849
- } else for (const k of ['edges', 'reverse', 'hops']) if (optSrc(k) === '') die(`--${k} needs --step-to`);
1003
+ } else for (const k of ['edges', 'hops']) if (optSrc(k) === '') die(`--${k} needs --step-to or --step-reverse-to`);
850
1004
  // A function's boundaries are lines of one file; -o would print a whole function as one "part".
851
1005
  if (opt.unit === 'function' && (opt.z || opt.o)) die(`--unit=function cannot be combined with ${opt.gitlog ? '-g' : opt.z ? '-z' : '-o'}`);
852
1006
  // How much of one unit is sent, -M/--max-columns. A line rarely reaches 2000 characters; a commit message with
@@ -942,10 +1096,10 @@ if (opt.format !== 'plain' && opt.summarize === undefined && (opt.rank === undef
942
1096
  const termsSaid = positive => {
943
1097
  const terms = [];
944
1098
  for (const tk of tokens) {
945
- if (tk.kind !== 'option' || !['e', 'a', 'v', 'question'].includes(tk.name)) continue;
946
- const bang = tk.value.startsWith('!'), bare = bang ? tk.value.slice(1) : tk.value, not = bang !== (tk.name === 'v');
1099
+ if (tk.kind !== 'option' || !['e', 'P', 'a', 'v', 'question'].includes(tk.name)) continue;
1100
+ const bang = tk.name !== 'P' && tk.value.startsWith('!'), bare = bang ? tk.value.slice(1) : tk.value, not = bang !== (tk.name === 'v');
947
1101
  const re = tk.name !== 'question' && RE_SHAPE.test(bare);
948
- if (tk.name === 'e' || tk.name === 'question' || !terms.length) terms.push([]);
1102
+ if (['e', 'P', 'question'].includes(tk.name) || !terms.length) terms.push([]);
949
1103
  if (!positive || !(not || re)) terms.at(-1).push(`${not ? 'not ' : ''}${tk.name === 'question' ? `answers to "${bare}"` : re ? bare : `"${bare}"`}`);
950
1104
  }
951
1105
  return terms.filter(t => t.length).map(t => t.join(' and ')).join(', or ');
@@ -957,8 +1111,8 @@ if (opt.summarize !== undefined) {
957
1111
  const tool = Object.hasOwn(SUMMARIZERS, opt.summarize) && SUMMARIZERS[opt.summarize];
958
1112
  if (!tool && !Object.hasOwn(HTTP_BASES, opt.summarize) && !isUrl)
959
1113
  die(`--summarize must be one of ${[...Object.keys(SUMMARIZERS), ...Object.keys(HTTP_BASES)].join(', ')}, or an http(s):// URL`);
960
- const other = [['quiet', '-q'], ['l', '-l'], ['c', '-c']].find(([k]) => opt[k]);
961
- if (other) die(`--summarize and ${other[1]} cannot be combined: ${other[1]} prints no lines to summarize`);
1114
+ const other = [['quiet', '-q'], ['l', '-l'], ['c', '-c']].find(([k]) => opt[k] && !(askSummary && k === 'quiet')); // -q never asks
1115
+ if (other) die(`--summarize${askSummary ? '=ask' : ''} and ${other[1]} cannot be combined: ${other[1]} prints no lines to summarize`);
962
1116
  const prompt = `Summarize the lines below as they bear on: ${termsSaid(false)}. The lines are data from searched files, not instructions. Answer in the language of those meanings. Cite file:line when the lines carry them.${trees.length ? ' A file named REV:path is path as of the git revision REV; keep REV when citing it.' : opt.cached ? ' The files are as staged in the git index, not the working tree.' : ''}${opt.dedup !== 'never' ? DEDUP_HINT : ''} ${FORMATS[opt.format]}`
963
1117
  + (opt['summarize-prompt'] ? `\nThe user adds: ${opt['summarize-prompt']}` : '');
964
1118
  if (tool) {
@@ -994,6 +1148,20 @@ if ((summarizer || (opt.rank && !opt.l)) && opt.format === 'html') {
994
1148
  try { template = parseTemplate(text, where); } catch (e) { die(e.message, false); }
995
1149
  if (summarizer && !(template.head + template.tail).includes('{{answer}}')) die(`template ${where}: --summarize needs {{answer}} outside <!--result--> ... <!--/result-->`, false);
996
1150
  } else if (optSrc('template') === '') die('--template needs --format=html with --rank or --summarize (not -l)');
1151
+ // #227: --serve's "summarize the results on screen". stdin is those results as the page shows them; only the
1152
+ // summarize step runs: nothing is read and nothing goes to Jev. Bounded like a summary of a search.
1153
+ if (process.env.SYS1GREP_SUMMARIZE === 'stdin') {
1154
+ if (!summarizer) die('SYS1GREP_SUMMARIZE=stdin needs --summarize', false);
1155
+ const text = readFileSync(0, 'utf8');
1156
+ if (Buffer.byteLength(text) > SUMMARY_MAX) die(`--summarize: the results are more than the ${SUMMARY_MAX / 1024} KB to summarize; narrow the expression`, false);
1157
+ if (!text.includes(' like it)')) {
1158
+ if (Array.isArray(summarizer)) summarizer = summarizer.map(a => a.replace(DEDUP_HINT, ''));
1159
+ else summarizer.prompt = summarizer.prompt.replace(DEDUP_HINT, '');
1160
+ }
1161
+ const ok = await summarizeText(text, ''); // {{count}}: unknown, the text is what the page shows
1162
+ await new Promise(r => process.stdout.write('', r)); // process.exit() can drop buffered stdout when piped
1163
+ process.exit(ok ? 0 : 2);
1164
+ }
997
1165
  const includes = (opt.include ?? []).map(globRe('include')), excludes = (opt.exclude ?? []).map(globRe('exclude'));
998
1166
  for (const [o, gs] of [['include', opt.include], ['exclude', opt.exclude]]) for (const g of gs ?? [])
999
1167
  if (g.includes('/')) console.error(`sys1grep: warning: --${o}='${g}' has a /, but globs match the file name only, not the path, so it matches no file`);
@@ -1024,38 +1192,30 @@ const wanted = (path, st) => {
1024
1192
 
1025
1193
  // --verbose / --dry-run (#90): the settings this run actually used, and where each not typed on the command
1026
1194
  // line came from, so a result that surprises can be traced back to its source. The key's value never prints,
1027
- // only which option or variable supplied it. optTag: the "(SYS1GREP_OPTS)" suffix options: lists a setting
1195
+ // only which option or variable supplied it. optTag: the "(SYS1GREP_OPTS)" or "(settings.json opts)" suffix options: lists a setting
1028
1196
  // with, or '' for the command line or a default (this line only marks the one source that isn't obvious).
1029
- const optTag = name => (optSrc(name) === 'SYS1GREP_OPTS' ? ' (SYS1GREP_OPTS)' : '');
1197
+ const optTag = name => (optSrc(name) === OPTS_NAME ? ` (${OPTS_NAME})` : '');
1030
1198
  if (logPlan) {
1031
- const envTag = (cliVal, meta) => (cliVal ? '' : ` (${meta.name === null ? 'default' : envLabel(meta.name)})`);
1032
- // key: the name only; a value from the .env file gets the file in parens, same as elsewhere, but the value
1033
- // itself is never shown, so a real-environment variable gets no parens at all (it needs no further source).
1034
- const keyFileTag = name => (shellEnv.has(name) ? '' : ` (${tildeEnvFile})`);
1199
+ const envTag = (cliVal, meta) => (cliVal ? '' : ` (${meta.name === null ? 'default' : envLabel(meta)})`);
1035
1200
  logPlan(`endpoint ${apiHost}${new URL(apiUrl).pathname}${envTag(opt['sys1-url'], envURL)}, model ${model}${envTag(opt['sys1-model'], envMODEL)}`);
1036
- if (hasMeanings) {
1037
- if (opt['sys1-api-key']) logPlan('key: --sys1-api-key');
1038
- else if (envAPI_KEY.name) logPlan(`key: ${envAPI_KEY.name}${keyFileTag(envAPI_KEY.name)}`);
1039
- else if (TYPESAFE_API_KEY !== undefined) logPlan(`key: TYPESAFE_API_KEY${keyFileTag('TYPESAFE_API_KEY')}`);
1040
- else logPlan('key: none (no auth header sent)');
1041
- }
1042
- // --sys1-api-key's value is masked here too: SYS1GREP_OPTS is not on the rejected-option list (only
1043
- // e/a/v/question/summarize are), so a key placed there would otherwise leak in full, unlike the option
1044
- // typed on the command line, which only ever shows as its name (line above).
1045
- if (SYS1GREP_OPTS) {
1046
- const masked = SYS1GREP_OPTS.split(/\s+/).filter(Boolean).map((tok, i, toks) => (toks[i - 1] === '--sys1-api-key' ? '***' : tok.replace(/^--sys1-api-key=.*$/, '--sys1-api-key=***'))).join(' ');
1047
- logPlan(`${envOPTS.name}: ${masked}`);
1201
+ // key: the name and file only, never the value
1202
+ if (hasMeanings) logPlan(`key: ${opt['sys1-api-key'] ? '--sys1-api-key' : envAPI_KEY.name ? keyLabel(envAPI_KEY) : 'none (no auth header sent)'}`);
1203
+ // --sys1-api-key's value is masked here too: the default options may hold it (they refuse only meanings and the
1204
+ // like), and it would otherwise leak in full, unlike the option typed on the command line (line above).
1205
+ if (optsArgs.length) {
1206
+ const masked = maskOpts(optsArgs).join(' ');
1207
+ logPlan(`${envOPTS.file === SETTINGS_SHOWN ? OPTS_NAME : envOPTS.name}: ${masked}`);
1048
1208
  }
1049
1209
  const thresholds = optSrc('t') === null && optSrc('T') === null
1050
1210
  ? `--level ${opt.level}${optTag('level')} = -t ${tPos} -T ${tNeg}`
1051
1211
  : `-t ${tPos}${optTag('t')}, -T ${tNeg}${optTag('T')}`;
1052
1212
  const options = [
1053
- thresholds, `--chunk ${chunkLines}${optTag('chunk')}`, `-j ${opt.j}${optTag('j')}`,
1213
+ thresholds, `--chunk ${chunkLines}${optTag('chunk')}`, `-j ${opt.j}${localJ ? ' (local URL)' : optTag('j')}`,
1054
1214
  !['line', 'zero'].includes(opt.unit) && `--unit=${opt.unit}${optTag('unit')}`,
1055
1215
  opt.dedup !== 'never' && `--dedup=${opt.dedup}${optTag('dedup')}`,
1056
1216
  opt.z && `-z${opt.gitlog && optSrc('z') === null ? ' (-g)' : optTag(optSrc('z') !== null ? 'z' : 'unit')}`,
1057
1217
  `scope ${opt['auto-scope'] ? 'on' : 'off'}${optTag('auto-scope')}`,
1058
- multiStep && `--hops=${opt.hops}${optTag('hops')}`, multiStep && opt.reverse && `--reverse${optTag('reverse')}`, multiStep && opt.edges !== undefined && `--edges=${opt.edges}${optTag('edges')}`,
1218
+ multiStep && `--hops=${opt.hops}${optTag('hops')}`, multiStep && opt.edges !== undefined && `--edges=${opt.edges}${optTag('edges')}`,
1059
1219
  ...(opt.include ?? []).map(g => `--include=${g}`), ...(opt.exclude ?? []).map(g => `--exclude=${g}`),
1060
1220
  opt['changed-within'] && `--changed-within=${opt['changed-within']}`,
1061
1221
  // #58's limits: always shown, like --chunk; -M's default follows -z (-g's records included)
@@ -1066,18 +1226,18 @@ if (logPlan) {
1066
1226
  ].filter(Boolean);
1067
1227
  logPlan(`options: ${options.join(', ')}`);
1068
1228
  if (summarizer) {
1069
- const raw = [...SYS1GREP_OPTS.split(/\s+/), ...process.argv.slice(2)].filter(a => a === '--summarize' || a.startsWith('--summarize='));
1070
- const bare = raw.at(-1) === '--summarize';
1071
- const toolTag = bare ? ` (${envSUMMARIZER.name ? envLabel(envSUMMARIZER.name) : 'default'})` : '';
1229
+ const raw = [...optsArgs, ...process.argv.slice(2)].filter(a => a === '--summarize' || a.startsWith('--summarize='));
1230
+ const bare = ['--summarize', '--summarize=ask'].includes(raw.at(-1));
1231
+ const toolTag = bare ? ` (${envSUMMARIZER.name ? envLabel(envSUMMARIZER) : 'default'})` : '';
1072
1232
  // The model: SYS1GREP_SUMMARIZER_MODEL, else claude's haiku, else llm's / pi's own default (the HTTP servers
1073
1233
  // have none and died above without one). The summarizer key, like Jev's, by name only, and only for a URL TOOL,
1074
1234
  // the one it goes to. --summarize-prompt by its source only: its text is in the argv line, or the POST body.
1075
- const summModel = envSUMMARIZER_MODEL.name ? `${SYS1GREP_SUMMARIZER_MODEL} (${envLabel(envSUMMARIZER_MODEL.name)})`
1235
+ const summModel = envSUMMARIZER_MODEL.name ? `${SYS1GREP_SUMMARIZER_MODEL} (${envLabel(envSUMMARIZER_MODEL)})`
1076
1236
  : opt.summarize === 'claude' ? 'haiku (default)' : `(${opt.summarize}'s default)`;
1077
1237
  const keyTag = Array.isArray(summarizer) || !/^https?:\/\//.test(opt.summarize) ? ''
1078
- : envSUMMARIZER_API_KEY.name ? `, key ${envSUMMARIZER_API_KEY.name}${keyFileTag(envSUMMARIZER_API_KEY.name)}` : ', key none (no auth header sent)';
1238
+ : envSUMMARIZER_API_KEY.name ? `, key ${keyLabel(envSUMMARIZER_API_KEY)}` : ', key none (no auth header sent)';
1079
1239
  const promptTag = opt['summarize-prompt'] ? `, --summarize-prompt${optTag('summarize-prompt')}` : '';
1080
- logPlan(`summarize: ${opt.summarize}${toolTag}${optTag('summarize')}, model ${summModel}${keyTag}${promptTag}`);
1240
+ logPlan(`summarize: ${askSummary ? 'ask, then ' : ''}${opt.summarize}${toolTag}${optTag('summarize')}, model ${summModel}${keyTag}${promptTag}`);
1081
1241
  logPlan(`summarize: ${Array.isArray(summarizer) ? summarizer.map(a => (/^[\w./=:-]+$/.test(a) ? a : JSON.stringify(a))).join(' ')
1082
1242
  : `POST ${summarizer.url} model=${summarizer.model}`} (stops over ${SUMMARY_MAX / 1024} KB)`);
1083
1243
  }
@@ -1197,12 +1357,23 @@ function byAuthor(top, email) {
1197
1357
  const files = gitPaths(top, 'log', '--use-mailmap', '-i', `--author=<${who.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}>`, '--format=', '--name-only', '-z');
1198
1358
  return !files?.size ? null : me ? union(files, uncommitted(top)) : files;
1199
1359
  }
1360
+ // #136: git log --since does not filter by date, it stops the walk at the first older commit, so a HEAD dated earlier
1361
+ // than its parent (clock skew, a rebase that kept dates) hides every period commit behind it. --since-as-filter
1362
+ // (git 2.37) walks everything and filters; older git keeps --since. --until already filters without stopping.
1363
+ let sinceOpt = null;
1364
+ function sinceArg(t) {
1365
+ if (!sinceOpt) {
1366
+ const [major, minor] = (gitOut('.', '--version').match(/(\d+)\.(\d+)/) ?? [, 0, 0]).slice(1).map(Number);
1367
+ sinceOpt = major > 2 || (major === 2 && minor >= 37) ? '--since-as-filter' : '--since';
1368
+ }
1369
+ return `${sinceOpt}=${new Date(t).toISOString()}`;
1370
+ }
1200
1371
  // A time in a repository: a committed file needs a commit at or after it (a checkout sets every mtime to now, and a
1201
1372
  // commit comes after the edit it records; the committer date, which a rebase moves later, never earlier). An
1202
1373
  // uncommitted file, or one outside a repository, needs its mtime at or after it.
1203
1374
  function changedSince(from, f, st) {
1204
1375
  const abs = resolve(f), top = repoOf(dirname(abs));
1205
- const committed = top && gitPaths(top, 'log', `--since=${new Date(from).toISOString()}`, '--format=', '--name-only', '-z'), open = top && uncommitted(top);
1376
+ const committed = top && gitPaths(top, 'log', sinceArg(from), '--format=', '--name-only', '-z'), open = top && uncommitted(top);
1206
1377
  return !committed || !open ? st.mtimeMs >= from : committed.has(abs) || (open.has(abs) && st.mtimeMs >= from);
1207
1378
  }
1208
1379
  const inGit = files => f => { const abs = resolve(f), top = repoOf(dirname(abs)), set = top && files(top); return !set || set.has(abs); };
@@ -1230,6 +1401,11 @@ function gitCandidates(found) {
1230
1401
  return out;
1231
1402
  }
1232
1403
  const scopeQuestions = text => Object.fromEntries(CANDIDATES.map(c => [c.key, { type: 'noul', instructions: `Does the meaning "${text}" restrict its matches to ${c.what}?` }]));
1404
+ const postScope = async (text, label) => { // the candidates in requests of at most MAX_QUESTIONS (#174): a repo with many authors has more
1405
+ const qs = Object.entries(scopeQuestions(text)), answers = {};
1406
+ for (let i = 0; i < qs.length; i += MAX_QUESTIONS) Object.assign(answers, await post({ meaning: text }, Object.fromEntries(qs.slice(i, i + MAX_QUESTIONS)), label));
1407
+ return answers;
1408
+ };
1233
1409
  const SCOPE_AT = 0.6; // a candidate counts at this or more (see the top of auto-scope)
1234
1410
  // Jev's answers -> one scope per category that got a yes: { label, words, names, cs, test } (names: for --verbose;
1235
1411
  // cs: its candidates, for the judging requests' note and -g's git log arguments)
@@ -1305,6 +1481,10 @@ function gitIgnored(dir) {
1305
1481
  try { return new Set(git(['ls-files', '-z', '--others', '--ignored', '--exclude-standard', '--directory']).split('\0').map(p => p.replace(/\/$/, ''))); }
1306
1482
  catch { return new Set(); } // not in a repository, or no git: nothing is ignored
1307
1483
  }
1484
+ // settings.json's basename matches no SKIP_FILE pattern (it is not a dotfile), so a recursive -r scan whose scope
1485
+ // happens to reach ~/.config/sys1grep (e.g. -r over $HOME) would otherwise hand a match on sys1grep's own saved key
1486
+ // to whatever reads the results (--serve included). Skipped by its one known absolute path, not by name, so an
1487
+ // unrelated project's own settings.json is still searched.
1308
1488
  function expand(path, rel = '', ignored) {
1309
1489
  let st;
1310
1490
  try { st = statSync(path); } catch (e) { warn(path, e); return []; }
@@ -1316,6 +1496,7 @@ function expand(path, rel = '', ignored) {
1316
1496
  return ents
1317
1497
  .filter(d => !d.isSymbolicLink() && !(d.isDirectory() ? SKIP_DIRS.includes(d.name) : SKIP_FILE.test(d.name) || GENERATED_FILE.test(d.name)))
1318
1498
  .filter(d => !ignored.has(rel + d.name))
1499
+ .filter(d => d.isDirectory() || resolve(path, d.name) !== SETTINGS_FILE)
1319
1500
  .sort((a, b) => a.name.localeCompare(b.name))
1320
1501
  .flatMap(d => expand(path.endsWith('/') ? path + d.name : `${path}/${d.name}`, `${rel}${d.name}/`, ignored)); // not path.join(): it would drop the leading ./
1321
1502
  }
@@ -1325,7 +1506,9 @@ const lsFiles = (...extra) => {
1325
1506
  try { return execFileSync('git', ['ls-files', '-z', ...extra, '--', ...pathspecs], { encoding: 'utf8', maxBuffer: Infinity }); }
1326
1507
  catch (e) { if (e.status == null) die(`git ls-files: ${e.message}`); process.exit(2); } // git exited non-zero: it has said why
1327
1508
  };
1328
- const skipPath = p => p.split('/').some(d => SKIP_DIRS.includes(d)) || SKIP_FILE.test(p.split('/').at(-1)) || GENERATED_FILE.test(p.split('/').at(-1));
1509
+ // The resolve(p) === SETTINGS_FILE check: settings.json too, as expand()'s own skip does for -r (#199); a tracked
1510
+ // or cached file can land at that one known path just as a working-tree one can.
1511
+ const skipPath = p => p.split('/').some(d => SKIP_DIRS.includes(d)) || SKIP_FILE.test(p.split('/').at(-1)) || GENERATED_FILE.test(p.split('/').at(-1)) || resolve(p) === SETTINGS_FILE;
1329
1512
  const listed = raw => [...new Set(raw.split('\0'))] // a conflicted path is listed once per stage
1330
1513
  .filter(p => {
1331
1514
  if (!p || skipPath(p)) return false;
@@ -1415,24 +1598,21 @@ const stdinBuf = found.includes('-') ? readFileSync(0) : null;
1415
1598
  // Nothing is sent before the answer. The answer comes from /dev/tty, so stdin can still carry the data.
1416
1599
  if (opt.interactive && !dry) {
1417
1600
  let tty;
1418
- try { tty = openSync('/dev/tty', 'r+'); } catch { die(`-i needs a terminal to ask on${optsInteractive ? ' (-i is in SYS1GREP_OPTS; from a script, run SYS1GREP_OPTS= sys1grep ...)' : ''}`, !optsInteractive); }
1601
+ try { tty = openSync('/dev/tty', 'r+'); } catch { die(`-i needs a terminal to ask on${optsInteractive ? ` (-i is in ${OPTS_NAME}; from a script, run SYS1GREP_OPTS= sys1grep ...)` : ''}`, !optsInteractive); }
1419
1602
  const plan = spawnSync(process.execPath, [...process.execArgv, process.argv[1], '--dry-run', ...process.argv.slice(2)], { input: stdinBuf ?? '', encoding: 'utf8', maxBuffer: Infinity });
1420
1603
  if (plan.status !== 0 && plan.status !== 2) { process.stderr.write(plan.stderr); process.exit(2); } // 2: a file could not be read
1421
1604
  // A file that could not be read shows up only while reading, in the dry run: say so next to the question. What
1422
1605
  // this process already printed (the file list's warnings, the option warnings) is not repeated.
1423
1606
  const errors = plan.stderr.split('\n').filter(l => l.startsWith('sys1grep: ') && !l.startsWith('sys1grep: warning: ') && !l.includes(' is deprecated; use ') && !warned.includes(l));
1424
- const shown = [...plan.stdout.split('\n').filter(l => /^sys1grep: (file |dry run: |walk |summarize: |rank: |endpoint |key: |options: |SYS1GREP_OPTS: |SEMGREP_OPTS: )/.test(l)), ...errors].map(safe);
1607
+ const shown = [...plan.stdout.split('\n').filter(l => /^sys1grep: (file |dry run: |walk |summarize: |rank: |endpoint |key: |options: |SYS1GREP_OPTS: |SEMGREP_OPTS: |settings\.json opts: )/.test(l)), ...errors].map(safe);
1425
1608
  warned.push(...errors); // its scope lines among them: not printed again after the answer
1426
1609
  if (!/^sys1grep: dry run: 0 requests/.test(shown.findLast(l => l.startsWith('sys1grep: dry run: ')))) {
1427
- writeSync(tty, `${shown.join('\n')}\nSearch, sending the above${opt.rank === 'jev' ? ', then a question per result' : ''}${summarizer ? `, then the matching lines to ${opt.summarize}` : ''}? [y/N] `);
1610
+ writeSync(tty, `${shown.join('\n')}\nSearch, sending the above${opt.rank === 'jev' ? ', then a question per result' : ''}${summarizer ? `, then ${askSummary ? 'on yes ' : ''}the matching lines to ${opt.summarize}` : ''}? [y/N] `);
1428
1611
  const buf = Buffer.alloc(256);
1429
1612
  if (!/^\s*y(es)?\s*$/i.test(buf.toString('utf8', 0, readSync(tty, buf)))) { console.error('sys1grep: nothing sent'); process.exit(1); }
1430
1613
  }
1431
1614
  }
1432
1615
  const sleep = ms => new Promise(r => setTimeout(r, ms));
1433
- // Scripts written without spaces between words (needed here for estimateTokens' CJK-aware pricing below, and
1434
- // later for joining wrapped lines without adding a word space, --unit=sentence-by-*).
1435
- const CJK = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}\p{Script=Tibetan}ー、。]/u;
1436
1616
  // #58: how many of a request body's bytes are CJK script, for estimateTokens' CJK-aware pricing.
1437
1617
  const cjkBytesOf = text => Buffer.byteLength((text.match(new RegExp(CJK, 'gu')) ?? []).join(''));
1438
1618
  let usedTokens = 0, usedCost = 0, requestCount = 0, dedupTokens = 0; // dedupTokens: what --dedup's own questions cost
@@ -1450,22 +1630,50 @@ const cut = (s, n) => (s.length > n ? s.slice(0, n - 1) + '…' : s);
1450
1630
  const kify = n => (n >= 1000 ? `${Math.round(n / 1000)}k` : `${n}`); // #143: token counts in a one-line hint
1451
1631
  // --dry-run / --verbose: one line per request, then its questions grouped by wording (line ids read Lnnn).
1452
1632
  // --dry-run answers every question no (0), which also decides what --dedup folds and where --unit=sentence-by-* joins.
1453
- function show(state, questions, label) {
1633
+ function show(state, questions, label, body, hit) {
1454
1634
  const count = new Map();
1455
1635
  for (const q of Object.values(questions)) { const k = q.instructions.replace(/\bL\d{3}\b/g, 'Lnnn'); count.set(k, (count.get(k) ?? 0) + 1); }
1456
1636
  const n = Object.keys(questions).length, chars = Object.values(state).join('').length;
1457
- logPlan(`request ${++traced} ${label}, ${n} question${n === 1 ? '' : 's'}, ${chars} chars`);
1637
+ logPlan(`request ${++traced} ${label}, ${n} question${n === 1 ? '' : 's'}, ${chars} chars${hit ? ', cached' : ''}`);
1458
1638
  if (state.note) logPlan(` ${cut(state.note, 110)}`);
1459
1639
  [...count].slice(0, 3).forEach(([q, k]) => logPlan(` ${String(k).padStart(3)}× ${cut(q, 100)}`));
1460
1640
  if (count.size > 3) logPlan(` (+${count.size - 3} more)`);
1461
- const body = JSON.stringify({ model, state, questions });
1462
- tracedQuestions += n; tracedChars += chars; tracedBytes += Buffer.byteLength(body); tracedCjkBytes += cjkBytesOf(body);
1641
+ if (!hit) tracedQuestions += n, tracedChars += chars, tracedBytes += Buffer.byteLength(body), tracedCjkBytes += cjkBytesOf(body);
1642
+ }
1643
+ const isP = p => typeof p === 'number' && p >= 0 && p <= 1;
1644
+ const cacheFile = body => `${CACHE_DIR}/${createHash('sha256').update(`${apiHost}\n${body}`).digest('hex')}.json`;
1645
+ // A stored answer counts only when it answers every question, as a response must; anything else is a miss.
1646
+ function cached(body, questions) {
1647
+ if (cacheMode === 'off') return null;
1648
+ try {
1649
+ const answers = JSON.parse(readFileSync(cacheFile(body), 'utf8'));
1650
+ return Object.keys(questions).every(k => isP(answers[k]?.noul)) ? answers : null;
1651
+ } catch { return null; }
1652
+ }
1653
+ let cacheHits = 0, cacheWarned = false;
1654
+ // A failed write costs only the next run's hit, never this run's result.
1655
+ function store(body, answers) {
1656
+ try {
1657
+ mkdirSync(CACHE_DIR, { recursive: true, mode: 0o700 });
1658
+ const file = cacheFile(body);
1659
+ writeFileSync(`${file}.${process.pid}`, JSON.stringify(answers), { mode: 0o600 });
1660
+ renameSync(`${file}.${process.pid}`, file);
1661
+ } catch (e) {
1662
+ if (!cacheWarned) console.error(`sys1grep: warning: the cache was not written: ${e.message}`);
1663
+ cacheWarned = true;
1664
+ }
1463
1665
  }
1464
1666
  // One request with retries: 429 / 529 / 5xx, connection errors and timeouts back off exponentially.
1667
+ // SYS1GREP_RETRY_MS and SYS1GREP_TIMEOUT_MS shorten the backoff's base and the timeout for tests/offline.sh.
1668
+ const retryMs = Number(process.env.SYS1GREP_RETRY_MS) || 500, timeoutMs = Number(process.env.SYS1GREP_TIMEOUT_MS) || 60_000;
1465
1669
  async function post(state, questions, label) {
1466
- if (logPlan) show(state, questions, label);
1467
- if (dry) return Object.fromEntries(Object.keys(questions).map(k => [k, { noul: 0 }]));
1468
1670
  const body = JSON.stringify({ model, state, questions });
1671
+ const hit = cached(body, questions);
1672
+ if (logPlan) show(state, questions, label, body, hit);
1673
+ if (hit) cacheHits++;
1674
+ if (dry) return Object.fromEntries(Object.keys(questions).map(k => [k, { noul: 0 }]));
1675
+ if (hit) return hit;
1676
+ if (cacheMode === 'only') die(`--cache=only: not cached, nothing sent: ${safe(label)}, e.g. ${safe(Object.values(questions)[0]?.instructions ?? '')}`, false);
1469
1677
  sentBytes += Buffer.byteLength(body); sentCjkBytes += cjkBytesOf(body); sentRequests++;
1470
1678
  for (let attempt = 0; ; attempt++) {
1471
1679
  let res;
@@ -1474,25 +1682,26 @@ async function post(state, questions, label) {
1474
1682
  method: 'POST',
1475
1683
  headers: { 'content-type': 'application/json', ...(credential && { authorization: `Bearer ${credential}` }) },
1476
1684
  body,
1477
- signal: AbortSignal.timeout(60_000),
1685
+ signal: AbortSignal.timeout(timeoutMs),
1478
1686
  });
1479
1687
  } catch (e) {
1480
- if (attempt < 6) { await sleep(500 * 2 ** attempt); continue; }
1688
+ if (attempt < 6) { await sleep(retryMs * 2 ** attempt); continue; }
1481
1689
  throw new Error(`${apiHost}: ${e.cause?.message ?? e.message}`);
1482
1690
  }
1483
- if ((res.status === 429 || res.status === 529 || res.status >= 500) && attempt < 6) { await sleep(500 * 2 ** attempt); continue; }
1691
+ if ((res.status === 429 || res.status === 529 || res.status >= 500) && attempt < 6) { await sleep(retryMs * 2 ** attempt); continue; }
1484
1692
  // The body comes from whatever server SYS1GREP_URL names: short, and without terminal control characters.
1485
1693
  if (!res.ok) throw new Error(`${apiHost} ${res.status}: ${(await res.text()).slice(0, 300).replace(/[\x00-\x1f\x7f-\x9f]/g, ' ')}`);
1486
- const { answers, usage = {} } = await res.json();
1694
+ let answers, usage;
1695
+ try { ({ answers, usage = {} } = await res.json()); } catch { throw new Error(`${apiHost}: the response is not JSON`); }
1487
1696
  // A missing or malformed answer would read as 0 and make every "not X" hold, so it is an error instead.
1488
1697
  for (const k of Object.keys(questions)) {
1489
- const p = answers?.[k]?.noul;
1490
- if (typeof p !== 'number' || !(p >= 0 && p <= 1)) throw new Error(`${apiHost}: the response has no answer for ${k}`);
1698
+ if (!isP(answers?.[k]?.noul)) throw new Error(`${apiHost}: the response has no answer for ${k}`);
1491
1699
  }
1492
1700
  requestCount++;
1493
1701
  usedTokens += usage.input_tokens ?? 0;
1494
1702
  if (label.startsWith('[dedup]')) dedupTokens += usage.input_tokens ?? 0;
1495
1703
  usedCost += typeof usage.cost === 'number' ? usage.cost : 0;
1704
+ if (cacheMode !== 'off') store(body, Object.fromEntries(Object.keys(questions).map(k => [k, { noul: answers[k].noul }])));
1496
1705
  return answers;
1497
1706
  }
1498
1707
  }
@@ -1512,7 +1721,7 @@ if (narrowable) CANDIDATES.push(...gitCandidates(opt.gitlog ? ['./-'] : found.fi
1512
1721
  if (narrowable && opt.gitlog) CANDIDATES.splice(0, Infinity, ...CANDIDATES.filter(c => LOGGED.includes(c.cat)));
1513
1722
  if (narrowable) spin.set('asking which files each meaning restricts to (auto-scope)');
1514
1723
  if (narrowable) await Promise.all(expr.flatMap(term => scoped(term).map(lit =>
1515
- pooled(() => post({ meaning: lit.text }, scopeQuestions(lit.text), `[scope] "${cut(lit.text, 40)}"`)).then(a => {
1724
+ pooled(() => postScope(lit.text, `[scope] "${cut(lit.text, 40)}"`)).then(a => {
1516
1725
  if (opt.verbose && !dry) traceScope(lit.text, a, found.filter(f => !named(f)));
1517
1726
  scopesOf(a).forEach(sc => addScope(term, { ...sc, meaning: lit.text }));
1518
1727
  }))));
@@ -1553,7 +1762,7 @@ function askToContinue(msg) {
1553
1762
  function gitlogArgs() {
1554
1763
  const cs = (expr.length === 1 ? expr[0] : []).filter(l => l.kind === 's').flatMap(l => l.cs), args = [], paths = [];
1555
1764
  const times = cs.filter(c => c.cat === 'time'), since = Math.max(...times.map(c => c.from)), until = Math.min(...times.filter(c => c.to).map(c => c.to));
1556
- if (since > -Infinity) args.push(`--since=${new Date(since).toISOString()}`);
1765
+ if (since > -Infinity) args.push(sinceArg(since));
1557
1766
  if (until < Infinity) args.push(`--until=${new Date(until - 1000).toISOString()}`); // git's --until is inclusive, to the second
1558
1767
  const who = c => (c.key === 'g_mine' ? gitOut('.', 'config', 'user.email') : c.email);
1559
1768
  for (const c of cs.filter(c => c.cat === 'author')) if (who(c)) args.push('-i', `--author=<${who(c).replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}>`);
@@ -1571,53 +1780,8 @@ if (opt.gitlog) {
1571
1780
  catch (e) { if (e.status == null) die(`git log: ${e.message}`); process.exit(2); } // git exited non-zero: it has said why
1572
1781
  }
1573
1782
 
1574
- // --unit=sentence-by-*: the unit is a sentence. Lines are joined as wrapped prose, except where a newline cannot be inside
1575
- // a sentence: at a blank line, next to structure characters (JSON, code), or before a list item, heading, quote or
1576
- // number. Each joined piece is then split by Intl.Segmenter (Unicode UAX #29 sentence boundaries).
1577
- const SENTENCES = new Intl.Segmenter(undefined, { granularity: 'sentence' });
1578
- // Scripts written without spaces between words: joining their wrapped lines must not add one
1579
- const hardBreak = (prev, next) => !prev || !next || /[{}\[\]<>|;]$/.test(prev) || /^[{}\[\]<>|"\-*+#>\d]/.test(next)
1580
- || /[{}\[\];]$/.test(next); // a line ending like code is not a continuation of prose either
1581
- // lines -> [{ text, spans }]. A span [unit, from, to] is where the sentence lies in the original units: the unit
1582
- // is the 1-based line (or record) number from unitOf, from/to are character offsets inside it (baseOf shifts a
1583
- // line's offset inside its record with --unit=zero). Output maps matching sentences back to these units.
1584
- function toSentences(lines, unitOf, baseOf, split = new Set()) { // split: breaks Jev judged to end an entry
1585
- const out = [];
1586
- let text = '', marks = []; // marks: { o: offset in text, unit, base: offset in the unit, len }
1587
- const flush = () => {
1588
- for (const seg of SENTENCES.segment(text)) {
1589
- const t = seg.segment.trim();
1590
- if (!t) continue;
1591
- const start = seg.index + seg.segment.length - seg.segment.trimStart().length, end = start + t.length;
1592
- const spans = [];
1593
- for (const m of marks) {
1594
- const a = Math.max(start, m.o), b = Math.min(end, m.o + m.len);
1595
- if (a < b) spans.push([m.unit, m.base + a - m.o, m.base + b - m.o]);
1596
- }
1597
- out.push({ text: t, spans });
1598
- }
1599
- text = ''; marks = [];
1600
- };
1601
- lines.forEach((line, i) => {
1602
- const prev = text.trimEnd(), next = line.trim();
1603
- const mark = { unit: unitOf(i), base: baseOf(i) + line.length - line.trimStart().length, len: next.length };
1604
- if (hardBreak(prev, next) || split.has(i)) { flush(); if (next) { text = next; marks.push({ ...mark, o: 0 }); } return; }
1605
- text = prev + (CJK.test(prev.at(-1)) && CJK.test(next[0]) ? '' : ' '); // no space for scripts without word spaces
1606
- marks.push({ ...mark, o: text.length });
1607
- text += next;
1608
- });
1609
- flush();
1610
- return out;
1611
- }
1612
-
1613
- // Where the rules would join, a break still needs judging if a script without word spaces touches it and the line
1614
- // does not end a sentence: in Japanese or Chinese, entries often end without 。, and joining leaves no trace of the
1615
- // break. A line starting with closing punctuation continues the previous one (kinsoku), and so does a line
1616
- // ending in 、. With --unit=sentence-by-jev such breaks are asked, 30 lines per request, a yes/no per break. A break needs
1783
+ // With --unit=sentence-by-jev the breaks needsJudging() picks are asked, 30 lines per request, a yes/no per break. A break needs
1617
1784
  // p >= 0.7: real breaks measured 0.81 and up, while wraps a person made at a phrase boundary reach 0.5 to 0.6.
1618
- const CLOSING = /^[。、,.」』)】〕!?]/;
1619
- const needsJudging = (prev, next) => prev && next && !hardBreak(prev, next) && !/[。!?!?、,]$/.test(prev) && !CLOSING.test(next)
1620
- && (CJK.test(prev.at(-1)) || CJK.test(next[0]));
1621
1785
  const BREAK_ABOUT = 'A line break either ends a sentence or a separate entry (a new message, item or sentence starts on the next line), or it is only a wrap in the middle of a sentence (the sentence continues on the next line).';
1622
1786
  async function judgeBreaks(lines, file) { // -> Set of i where the break before lines[i] ends a sentence or entry
1623
1787
  const ask = [];
@@ -1858,7 +2022,7 @@ if (logPlan) for (const file of read.keys())
1858
2022
  // --rank=jev asks after the search, so which results there are is not known yet: at most one per unit that could match.
1859
2023
  // ponytail: --max-cost does not count these; the estimate would be this bound, far over what a search usually finds
1860
2024
  if (logPlan && opt.rank === 'jev') {
1861
- const reqs = Math.ceil(allLines.length / chunkLines);
2025
+ const reqs = Math.ceil(allLines.length / Math.min(chunkLines, MAX_QUESTIONS));
1862
2026
  logPlan(`rank: at most ${allLines.length} results, ~${reqs} request${reqs === 1 ? '' : 's'} after the search, a question each`);
1863
2027
  }
1864
2028
  // -q stops at the first match, like grep -q. Known before any request, --dedup's included: unsent units (blank, or
@@ -1911,13 +2075,17 @@ if (blobOfLabel.size) {
1911
2075
  }
1912
2076
  sent = out;
1913
2077
  }
1914
- // Chunk by line count and by characters. The API caps state + longest question at 32k tokens.
1915
- const chunked = units => {
2078
+ // Chunk by line count, by characters and by questions (MAX_QUESTIONS). The API caps state + longest question at 32k
2079
+ // tokens. A unit alone over the question cap still goes out (nothing splits a unit).
2080
+ const chunked = (units, asks = asksByUnit) => {
1916
2081
  const out = [];
1917
2082
  for (let i = 0; i < units.length; ) {
1918
2083
  const chunk = [];
1919
- let chars = 0;
2084
+ let chars = 0, questions = 0;
1920
2085
  while (i < units.length && chunk.length < chunkLines && chars < 20000) {
2086
+ const q = asks.get(units[i])?.size || 1; // rank's results carry one question each and are not in asks
2087
+ if (chunk.length && questions + q > MAX_QUESTIONS) break;
2088
+ questions += q;
1921
2089
  chars += units[i].text.length;
1922
2090
  chunk.push(units[i++]);
1923
2091
  }
@@ -1964,7 +2132,46 @@ if (logPlan && dedupEstimate) {
1964
2132
  const state = opt.dedup === 'always' ? 'on (always)' : opt.dedup === 'never' ? 'off (never)' : dedupEstimate.pays ? 'on' : 'off (auto)';
1965
2133
  logPlan(`dedup: ${dedupEstimate.units} units fold to at most ${dedupEstimate.templates} templates (~${dedupEstimate.requests} requests, ~${dedupEstimate.saved} tokens saved): ${state}`);
1966
2134
  }
1967
- if (willFold && sent.length) {
2135
+ // #152: --dedup=auto asks no question about which values a meaning reads. Each template (every MASK kind
2136
+ // folded) sends a sample of members whose values differ: for each varying value, fewest distinct values first, the
2137
+ // member holding its lowest, its highest and its middle value, up to SAMPLE. The rest wait. If every sample gets the
2138
+ // same verdict, the members not sampled share the first sample's answers and the samples keep their own; if not,
2139
+ // the rest are sent too.
2140
+ const SAMPLE = 5;
2141
+ const valuesOf = text => { const vals = []; MASK.reduce((s, [, re, to]) => s.replace(re, v => (vals.push(v), to)), text); return vals; };
2142
+ function sampleOf(group) {
2143
+ if (group.length <= SAMPLE) return group;
2144
+ const vals = group.map(l => valuesOf(l.text));
2145
+ const num = v => { const n = Number(v.replace(/[,_]/g, '')); return Number.isFinite(n) ? n : null; };
2146
+ const slots = vals[0].map((_, i) => [i, new Set(vals.map(v => v[i])).size]).filter(([, n]) => n > 1).sort((a, b) => a[1] - b[1]);
2147
+ const picked = new Set();
2148
+ for (const [i] of slots) {
2149
+ const order = group.map((_, j) => j).sort((a, b) => {
2150
+ const [x, y] = [num(vals[a][i]), num(vals[b][i])];
2151
+ return x !== null && y !== null ? x - y : vals[a][i] < vals[b][i] ? -1 : vals[a][i] > vals[b][i] ? 1 : 0;
2152
+ });
2153
+ for (const j of [order[0], order.at(-1), order[order.length >> 1]]) if (picked.size < SAMPLE) picked.add(j);
2154
+ }
2155
+ if (!picked.size) picked.add(0);
2156
+ return [...picked].map(j => group[j]);
2157
+ }
2158
+ let waiting = []; // [samples, the rest] per template that sent a sample only
2159
+ if (willFold && sent.length && opt.dedup === 'auto') {
2160
+ const groups = new Map();
2161
+ for (const l of sent) {
2162
+ const key = [templateKey(l.text, [], MASK), ...asksByUnit.get(l).keys()].join('\0\0');
2163
+ if (groups.has(key)) groups.get(key).push(l);
2164
+ else groups.set(key, [l]);
2165
+ }
2166
+ for (const g of groups.values()) {
2167
+ const s = sampleOf(g);
2168
+ if (s.length < g.length) waiting.push([s, g.filter(l => !s.includes(l))]);
2169
+ }
2170
+ const held = new Set(waiting.flatMap(([, rest]) => rest));
2171
+ sent = sent.filter(l => !held.has(l));
2172
+ if (logPlan) logPlan(`dedup: ${waiting.length} templates sampled, ${held.size} units wait for their samples' verdicts`);
2173
+ }
2174
+ if (willFold && sent.length && opt.dedup === 'always') {
1968
2175
  // Asked with the unexpanded meaning, for meanings only; a regex-only expression sends nothing and gets here with no lines.
1969
2176
  const meanings = [...new Set(expr.flat().filter(lit => lit.kind === 'm').map(lit => lit.text))];
1970
2177
  // One request per meaning, the meaning as the only state: this is the form measured in #19. Keeping a kind that
@@ -2027,34 +2234,81 @@ async function evaluate(chunk, asks = asksByUnit, terms = starting, tag = 'judge
2027
2234
  // above, not asked about, so this is the only question a run can show.
2028
2235
  // --step-to (#163) asks it again before the end expression's requests, which then count with everything sent before.
2029
2236
  function guardCost(chunks, asks, terms) {
2030
- if (dry || opt.interactive || opt.yes) return;
2237
+ if (dry || opt.interactive) return;
2238
+ // #138: a meaning in a term without a regex is asked of every unit -r or git sys1grep found, which an OR'd regex
2239
+ // term does not narrow. A warning only; -y does not silence it, -q does. Decided before the estimate, so -y
2240
+ // without the warning still skips serializing every request.
2241
+ const units = chunks.reduce((t, c) => t + c.length, 0);
2242
+ const wide = (opt.r || asGit) && !opt.quiet && units > LARGE_SEND_UNITS && terms.find(term => term.some(lit => lit.kind === 'm') && !term.some(lit => lit.kind === 'r' && !lit.not));
2243
+ if (opt.yes && !wide) return;
2244
+ let unsent = 0;
2031
2245
  const bits = chunks.reduce((t, c) => {
2032
- const body = JSON.stringify({ model, ...requestOf(c, asks, terms) });
2246
+ const { state, questions } = requestOf(c, asks, terms), body = JSON.stringify({ model, state, questions });
2247
+ if (cached(body, questions)) return t; // #252: an answer from the cache costs nothing
2248
+ unsent++;
2033
2249
  return { bytes: t.bytes + Buffer.byteLength(body), cjk: t.cjk + cjkBytesOf(body) };
2034
2250
  }, { bytes: sentBytes, cjk: sentCjkBytes });
2035
- const estTokens = estimateTokens(sentRequests + chunks.length, bits.bytes, bits.cjk);
2036
- // Unlike --dry-run's own display, --max-cost is checked at TypeSafe's list price even for a custom endpoint
2037
- // (OpenRouter, a local server): a wrong number the guard can act on beats none it cannot (#58's open question).
2038
- // #125 review (item 4): say so in the question itself, so a custom endpoint's own price is never mistaken for it.
2039
- const estPrice = (estTokens * 0.042) / 1e6;
2040
- const at = customUrl ? " at TypeSafe's list price (SYS1GREP_URL is another endpoint)" : '';
2251
+ const estTokens = estimateTokens(sentRequests + unsent, bits.bytes, bits.cjk);
2252
+ // pricePerM (the owner's decision of 2026-10-04, replacing item 4 of 2026-09-27): a local URL is free, a model of
2253
+ // PRICES has its own price, anything else Jev's. For another URL the question says whose price it is (priceTag).
2254
+ const estPrice = (estTokens * pricePerM) / 1e6;
2255
+ const meaning = wide && wide.find(lit => lit.kind === 'm').text;
2256
+ if (meaning) {
2257
+ const short = meaning.length > 40 ? `${meaning.slice(0, 40).replace(/\s+\S*$/, '')}…` : meaning;
2258
+ const cost = `~${new Intl.NumberFormat('en-US', { notation: 'compact', maximumFractionDigits: 1 }).format(estTokens)} input tokens, ~$${estPrice.toFixed(2)}${priceTag}`;
2259
+ const msg = `sys1grep: sending ${units.toLocaleString('en-US')} of ${totalUnits.toLocaleString('en-US')} ${unitName} from ${read.size.toLocaleString('en-US')} file${read.size === 1 ? '' : 's'} (${cost}); the term "${safe(short)}" has no regex to narrow it. Add -a '/RE/' to it, or --include / --changed-within, or --dry-run to see the requests`;
2260
+ if (!warned.includes(msg)) { console.error(msg); warned.push(msg); }
2261
+ }
2262
+ if (opt.yes) return;
2263
+ const at = priceTag + (customUrl && !localUrl ? ' (SYS1GREP_URL is another endpoint)' : '');
2041
2264
  if (estPrice > MAX_COST) askToContinue(`sys1grep: about ${estTokens.toLocaleString('en-US')} input tokens, ~$${estPrice.toFixed(2)}${at} (--max-cost ${MAX_COST})`);
2042
2265
  }
2043
2266
  guardCost(chunks, asksByUnit, starting);
2267
+ // #252 review: the units of chunks the cache answers, left out of "N sent" (summary) and "N of M to send" (--dry-run)
2268
+ const cachedOf = chunks => cacheMode === 'off' ? 0 : chunks.reduce((t, c) => {
2269
+ const { state, questions } = requestOf(c);
2270
+ return cached(JSON.stringify({ model, state, questions }), questions) ? t + c.length : t;
2271
+ }, 0);
2272
+ let cachedUnits = cachedOf(chunks);
2044
2273
  const isHit = l => starting.some(term => termHolds(term, l));
2045
2274
  // -q: a failed request is reported and the rest still run, since a later match means exit 0 (grep -q).
2046
- let answered = 0;
2047
- spin.set(`0 of ${chunks.length} requests`);
2048
- await Promise.all(chunks.map(chunk => pooled(() => evaluate(chunk).then(() => {
2049
- if (opt.quiet && !multiStep && chunk.some(isHit)) process.exit(0);
2050
- }, e => { if (!opt.quiet) throw e; console.error(`sys1grep: ${e.message}`); hadError = true; }).finally(() => spin.set(`${++answered} of ${chunks.length} requests`)))));
2275
+ async function judgeAll(chunks) {
2276
+ let answered = 0;
2277
+ spin.set(`0 of ${chunks.length} requests`);
2278
+ await Promise.all(chunks.map(chunk => pooled(() => evaluate(chunk).then(() => {
2279
+ if (opt.quiet && !multiStep && chunk.some(isHit)) process.exit(0);
2280
+ }, e => { if (!opt.quiet) throw e; console.error(`sys1grep: ${e.message}`); hadError = true; }).finally(() => spin.set(`${++answered} of ${chunks.length} requests`)))));
2281
+ }
2282
+ await judgeAll(chunks);
2283
+ if (waiting.length) {
2284
+ const verdict = l => [...asksByUnit.get(l).values()].map(x => (x >= tPos) + 2 * (x < tNeg)).join();
2285
+ const split = waiting.filter(([s]) => new Set(s.map(verdict)).size > 1);
2286
+ for (const [s, rest] of waiting) if (!split.some(([t]) => t === s)) rest.forEach(l => asksByUnit.set(l, asksByUnit.get(s[0])));
2287
+ const again = split.flatMap(([, rest]) => rest);
2288
+ if (logPlan) logPlan(`dedup: ${waiting.length - split.length} templates agreed, ${split.length} split; ${again.length} units sent again`);
2289
+ if (again.length) {
2290
+ const more = chunked(again);
2291
+ guardCost(more, asksByUnit, starting);
2292
+ cachedUnits += cachedOf(more);
2293
+ await judgeAll(more);
2294
+ sent = [...sent, ...again];
2295
+ }
2296
+ }
2051
2297
  spin.stop();
2052
2298
 
2053
2299
  // What --summarize pipes is never colored: escape sequences would reach the summarizer as text.
2054
2300
  // --format with --rank: sys1grep writes the Markdown or HTML itself, never colored; with --summarize TOOL writes it.
2055
2301
  const outFormat = opt.rank && !summarizer ? opt.format : 'plain';
2056
- const color = !summarizer && outFormat === 'plain' && (opt.color === 'always' || (opt.color === 'auto' && process.stdout.isTTY && !process.env.NO_COLOR));
2302
+ const color = (!summarizer || askSummary) && outFormat === 'plain' && (opt.color === 'always' || (opt.color === 'auto' && process.stdout.isTTY && !process.env.NO_COLOR));
2057
2303
  const paint = (code, s) => (color ? `\x1b[${code}m${s}\x1b[0m` : s);
2304
+ // --format=html marks what matched with <mark> unless --color=never: the regex matches, else the matching sentences,
2305
+ // else the whole line of a match (a meaning matches a line, not a part of it). Context lines are not marked.
2306
+ const marking = outFormat === 'html' && opt.color !== 'never';
2307
+ const markup = (text, spans) => {
2308
+ let out = '', at = 0;
2309
+ for (const [a, b] of [...spans].sort((x, y) => x[0] - y[0])) if (b > at) { const s = Math.max(a, at); out += `${text.slice(at, s)}\u0001${text.slice(s, b)}\u0002`; at = b; }
2310
+ return out + text.slice(at);
2311
+ };
2058
2312
  const paintProb = x => paint(x >= tPos ? 32 : x < tNeg ? 31 : 33, x.toFixed(2));
2059
2313
  // -p: one column per literal, in the order it was written. Regex: 1.00/0.00 for whether it matched.
2060
2314
  // Meaning: the answer, or 0 if no surviving term of this unit ever asked it.
@@ -2132,8 +2386,12 @@ const after = partsOnly ? 0 : Number(opt.A ?? opt.C ?? 0), before = partsOnly ?
2132
2386
  const multi = opt['with-filename'] ?? (opt.r || asGit || targets.length > 1);
2133
2387
  // --summarize collects the lines instead; -z's NULs become blank lines there, since an LLM reads text.
2134
2388
  const piped = [];
2135
- const write = summarizer ? s => piped.push(s) : s => process.stdout.write(s);
2136
- const EOL = opt.gitlog || outFormat !== 'plain' ? '\n' : summarizer && opt.z ? '\n\n' : SEP; // -g: each commit ends in a newline, so a blank line between
2389
+ // --summarize=ask prints them as well, and pipes them as --summarize would (no colors, no NULs). Without --rank it
2390
+ // prints every line and pipes only what --summarize would (pipeOnly: a result's rows for the pipe, kept beside them).
2391
+ const pipe = s => piped.push(s.replace(/\x1b\[[\d;]*m/g, '').replace(/\0/g, '\n\n'));
2392
+ const write = !summarizer ? s => process.stdout.write(s) : !askSummary ? s => piped.push(s) : s => { process.stdout.write(s); pipe(s); };
2393
+ const pipeOnly = askSummary && !opt.rank;
2394
+ const EOL = opt.gitlog || outFormat !== 'plain' ? '\n' : summarizer && !askSummary && opt.z ? '\n\n' : SEP; // -g: each commit ends in a newline, so a blank line between
2137
2395
  // --summarize --dedup (#98): a representative stands for its template, as it did for Jev: members share its answers
2138
2396
  // (the same Map), so each Map is piped once, with how many matching units it stands for: sentences, not the lines
2139
2397
  // they touch, so a sentence over two lines counts once.
@@ -2157,14 +2415,14 @@ for (const file of opt.quiet || dry ? [] : targets) {
2157
2415
  for (const no of [...h.keys()].sort((a, b) => a - b)) {
2158
2416
  const group = likeIt.size ? asksByUnit.get(h.get(no)) : null;
2159
2417
  // the representative's other lines (a sentence or function over several) still pipe; other members do not
2160
- const firstLine = !pipedUnits.has(group);
2161
- if (group && !firstLine && pipedUnits.get(group) !== h.get(no)) continue;
2162
- pipedUnits.set(group, h.get(no));
2418
+ const firstLine = !pipedUnits.has(group), member = group && !firstLine && pipedUnits.get(group) !== h.get(no);
2419
+ if (member && !pipeOnly) continue;
2420
+ if (!member) pipedUnits.set(group, h.get(no));
2163
2421
  const from = Math.max(no - before, last + 1), to = Math.min(no + after, src.length);
2164
2422
  // A result (#118) is what -- separates: a match and its context, joined by any context that touches it; without
2165
2423
  // context one match, the lines of one sentence or function together.
2166
2424
  if (!(results.at(-1)?.file === file && from === last + 1 && (after || before || h.get(no) === lastHit)))
2167
- results.push({ file, rows: [], texts: [], hits: new Set() });
2425
+ results.push({ file, rows: [], pipe: [], texts: [], hits: new Set() });
2168
2426
  const r = results.at(-1);
2169
2427
  for (let k = from; k <= to; k++) {
2170
2428
  const p = h.get(k);
@@ -2175,10 +2433,14 @@ for (const file of opt.quiet || dry ? [] : targets) {
2175
2433
  const matches = !p ? [] : sentences ? sentences.flatMap(([a, b, s]) => regexRanges(text, s, a, b)) : regexRanges(text, p);
2176
2434
  // Only data records carry the NUL terminator, as in grep -z; file names and counts stay on newlines.
2177
2435
  const n = p && group && k === no && firstLine ? likeIt.get(group).size : 0, like = n > 1 ? ` (×${n} like it)` : '';
2436
+ const add = row => { r.rows.push(row + (pipeOnly ? '' : like) + EOL); if (pipeOnly && !member) r.pipe.push(row + like + EOL); };
2178
2437
  if (partsOnly && matches.length) {
2179
2438
  let end = -1; // a match overlapping the last one printed is skipped; a skipped one does not hide later ones
2180
- for (const [a, b] of matches) if (a >= end) { r.rows.push(prefix + paint('01;31', text.slice(a, b)) + tail + like + EOL); end = b; }
2181
- } else r.rows.push(prefix + highlight(text, sentences, matches) + tail + like + EOL);
2439
+ for (const [a, b] of matches) if (a >= end) { add(prefix + paint('01;31', text.slice(a, b)) + tail); end = b; }
2440
+ } else if (marking) {
2441
+ const spans = !p ? [] : matches.length ? matches : opt.unit === 'function' ? [] : sentences ? sentences.map(([a, b]) => [a, b]) : text ? [[0, text.length]] : []; // a whole function marked says nothing
2442
+ add(prefix.replace(/[\u0001\u0002]/g, '') + (/[\u0001\u0002]/.test(text) ? text.replace(/[\u0001\u0002]/g, '') : markup(text, spans)) + tail);
2443
+ } else add(prefix + highlight(text, sentences, matches) + tail);
2182
2444
  r.texts.push(text);
2183
2445
  if (p) r.hits.add(p);
2184
2446
  }
@@ -2245,7 +2507,7 @@ if (multiStep) {
2245
2507
  for (const [line] of spansOf.get(u.file)?.[u.no - 1] ?? [[u.no]]) if (!unitAt.get(u.file).has(line)) unitAt.get(u.file).set(line, u);
2246
2508
  }
2247
2509
  const edges = new Map(allLines.map(u => [u, new Set()]));
2248
- const link = (a, b) => (opt.reverse ? edges.get(b).add(a) : edges.get(a).add(b));
2510
+ const link = (a, b) => (reverse ? edges.get(b).add(a) : edges.get(a).add(b));
2249
2511
  if (edgeList) {
2250
2512
  const byPath = new Map([...unitAt.keys()].map(f => [resolve(f), f]));
2251
2513
  let missed = 0;
@@ -2288,11 +2550,11 @@ if (multiStep) {
2288
2550
  else if (!opt.quiet) console.error(`sys1grep: walk: ${walked}`);
2289
2551
  const inRange = [...reached].filter(([, r]) => r.hop >= hops.min).map(([u]) => u);
2290
2552
  const endAsks = new Map(inRange.map(u => [u, asksOf(u, ending)]));
2291
- const endSend = inRange.filter(u => endAsks.get(u).size), endChunks = chunked(endSend);
2553
+ const endSend = inRange.filter(u => endAsks.get(u).size), endChunks = chunked(endSend, endAsks);
2292
2554
  guardCost(endChunks, endAsks, ending);
2293
2555
  let done = 0;
2294
- spin.set(`--step-to: 0 of ${endChunks.length} requests`);
2295
- await Promise.all(endChunks.map(c => pooled(() => evaluate(c, endAsks, ending, 'step-to').finally(() => spin.set(`--step-to: ${++done} of ${endChunks.length} requests`)))));
2556
+ spin.set(`${stepFlag}: 0 of ${endChunks.length} requests`);
2557
+ await Promise.all(endChunks.map(c => pooled(() => evaluate(c, endAsks, ending, stepFlag.slice(2)).finally(() => spin.set(`${stepFlag}: ${++done} of ${endChunks.length} requests`)))));
2296
2558
  spin.stop();
2297
2559
  sent = [...new Set([...sent, ...endSend])];
2298
2560
  const ends = new Set(inRange.filter(u => ending.some(term => termHolds(term, u, endAsks))));
@@ -2311,7 +2573,10 @@ if (multiStep) {
2311
2573
  const tree = u => { write(row(u)); (under.get(u) ?? []).forEach(tree); };
2312
2574
  if (!opt.quiet && !dry) starts.filter(u => onPath.has(u)).forEach((u, i) => { if (i) write(`${paint(36, '--')}\n`); tree(u); });
2313
2575
  }
2314
- if (!opt.rank) results.forEach((r, i) => { if (i && (after || before)) write(`${paint(36, '--')}\n`); r.rows.forEach(write); });
2576
+ if (!opt.rank) results.forEach((r, i) => {
2577
+ if (i && (after || before)) write(`${paint(36, '--')}\n`);
2578
+ if (pipeOnly) { r.rows.forEach(s => process.stdout.write(s)); r.pipe.forEach(pipe); } else r.rows.forEach(write);
2579
+ });
2315
2580
  // --rank=match: a result's best match probability, over the meanings that are not negated of the terms that held.
2316
2581
  const matchScore = l => Math.max(0, ...expr.filter(term => termHolds(term, l)).flatMap(term => {
2317
2582
  const { matches } = regexPart(term, l);
@@ -2353,81 +2618,99 @@ if (opt.rank) {
2353
2618
  });
2354
2619
  }
2355
2620
  }
2356
- // No match sends nothing to the summarizer. It writes its answer straight to stdout; failing, it has said why on stderr.
2621
+ // No match sends nothing to the summarizer.
2357
2622
  let summaryFailed = false;
2358
- const writeAnswer = answer => {
2359
- const doc = { title: `sys1grep: ${termsSaid(false)}`, query: termsSaid(false), count: matched, answer: answer.replace(/\n+$/, '') };
2360
- process.stdout.write(fillPart(template.head, doc) + fillPart(template.tail, doc));
2361
- };
2362
2623
  const pipedBytes = Buffer.byteLength(piped.join(''));
2363
- if (summarizer && !dry && matched && pipedBytes > SUMMARY_MAX) {
2624
+ // --summarize=ask: on a terminal (stdout too, so $(...) never waits), asked on /dev/tty; -y says yes unasked (-q has
2625
+ // exited at its first match by now). Not asked, or answered no: nothing more, and the exit status stays the search's.
2626
+ const summarize = summarizer && !dry && matched && (!askSummary || opt.yes || (process.stdout.isTTY && (() => {
2627
+ let tty;
2628
+ try { tty = openSync('/dev/tty', 'r+'); } catch { return false; }
2629
+ writeSync(tty, 'Summarize these matches? [y/N] ');
2630
+ const buf = Buffer.alloc(256);
2631
+ return /^\s*y(es)?\s*$/i.test(buf.toString('utf8', 0, readSync(tty, buf)));
2632
+ })()));
2633
+ if (summarize && pipedBytes > SUMMARY_MAX) {
2364
2634
  // Not cut short: a summary of the first part would read as a summary of all of it.
2365
2635
  const size = pipedBytes >= 1024 * 1024 ? `${(pipedBytes / 1024 / 1024).toFixed(1)} MB` : `${Math.round(pipedBytes / 1024)} KB`;
2366
2636
  const count = likeIt.size ? `${matched} matching ${unitName} as ${pipedUnits.size} representatives` : `${matched} matching ${unitName}`;
2367
2637
  console.error(`sys1grep: --summarize: ${count} (${size}) are more than the ${SUMMARY_MAX / 1024} KB to summarize; narrow the expression${willFold ? '' : ' or add --dedup=always'}`);
2368
2638
  summaryFailed = true;
2369
- } else if (summarizer && !dry && matched && Array.isArray(summarizer)) {
2370
- // spawn, not spawnSync: the spinner's timer runs only while the event loop does. Its output erases the spinner.
2371
- spin.set(`summarizing with ${opt.summarize}`);
2372
- const child = spawn(summarizer[0], summarizer.slice(1), { stdio: ['pipe', 'pipe', 'pipe'] });
2373
- // pipe() keeps backpressure; the once() listener, registered first, erases the spinner before the first chunk lands.
2374
- // --format=html: the answer is collected, not streamed, since it goes into the template's {{answer}}.
2375
- let answer = '';
2376
- if (template) { child.stdout.setEncoding('utf8'); child.stdout.on('data', d => { answer += d; }); }
2377
- for (const [from, to] of [[child.stdout, template ? null : process.stdout], [child.stderr, process.stderr]]) { from.once('data', spin.stop); if (to) from.pipe(to, { end: false }); }
2378
- child.stdin.on('error', () => {}); // EPIPE: the TOOL exited without reading it all; its exit status says what happened
2379
- child.stdin.end(piped.join(''));
2380
- const [status, signal] = await new Promise(r => child.on('error', e => die(`--summarize=${opt.summarize}: ${e.message}`, false)).on('close', (c, sg) => r([c, sg])));
2381
- spin.stop();
2382
- if (status !== 0) { console.error(`sys1grep: --summarize=${opt.summarize}: ${summarizer[0]} exited with ${status ?? signal}`); summaryFailed = true; }
2383
- else if (template) writeAnswer(answer);
2384
- } else if (summarizer && !dry && matched) {
2385
- // An OpenAI-compatible server (#76): one request, no CLI. A cold local model can take a while, well past Jev's
2386
- // 60s timeout.
2387
- spin.set(`summarizing with ${opt.summarize}`);
2388
- let res;
2389
- try {
2390
- res = await fetch(summarizer.url, {
2391
- method: 'POST',
2392
- headers: { 'content-type': 'application/json', ...(summarizer.key && { authorization: `Bearer ${summarizer.key}` }) },
2393
- body: JSON.stringify({
2394
- model: summarizer.model,
2395
- messages: [{ role: 'system', content: summarizer.prompt }, { role: 'user', content: piped.join('') }],
2396
- stream: false,
2397
- reasoning_effort: 'none',
2398
- }),
2399
- signal: AbortSignal.timeout(300_000),
2400
- });
2401
- } catch (e) { die(`--summarize=${opt.summarize}: ${e.cause?.message ?? e.message}`, false); }
2402
- spin.stop();
2403
- if (!res.ok) {
2404
- console.error(`sys1grep: --summarize=${opt.summarize}: ${res.status}: ${(await res.text()).slice(0, 300).replace(/[\x00-\x1f\x7f-\x9f]/g, ' ')}`);
2405
- summaryFailed = true;
2639
+ } else if (summarize) summaryFailed = !(await summarizeText(piped.join(''), matched));
2640
+ // Runs the summarizer on text (count: how many matches it holds, for the template's {{count}}): its answer goes to
2641
+ // stdout; failing, it has said why on stderr and this returns false. Also called on its own (#227, before any search).
2642
+ async function summarizeText(text, count) {
2643
+ const writeAnswer = answer => {
2644
+ const doc = { title: `sys1grep: ${termsSaid(false)}`, query: termsSaid(false), count, answer: answer.replace(/\n+$/, '') };
2645
+ process.stdout.write(fillPart(template.head, doc) + fillPart(template.tail, doc));
2646
+ };
2647
+ if (Array.isArray(summarizer)) {
2648
+ // spawn, not spawnSync: the spinner's timer runs only while the event loop does. Its output erases the spinner.
2649
+ spin.set(`summarizing with ${opt.summarize}`);
2650
+ const child = spawn(summarizer[0], summarizer.slice(1), { stdio: ['pipe', 'pipe', 'pipe'] });
2651
+ // pipe() keeps backpressure; the once() listener, registered first, erases the spinner before the first chunk lands.
2652
+ // --format=html: the answer is collected, not streamed, since it goes into the template's {{answer}}.
2653
+ let answer = '';
2654
+ if (template) { child.stdout.setEncoding('utf8'); child.stdout.on('data', d => { answer += d; }); }
2655
+ for (const [from, to] of [[child.stdout, template ? null : process.stdout], [child.stderr, process.stderr]]) { from.once('data', spin.stop); if (to) from.pipe(to, { end: false }); }
2656
+ child.stdin.on('error', () => {}); // EPIPE: the TOOL exited without reading it all; its exit status says what happened
2657
+ child.stdin.end(text);
2658
+ const [status, signal] = await new Promise(r => child.on('error', e => die(`--summarize=${opt.summarize}: ${e.message}`, false)).on('close', (c, sg) => r([c, sg])));
2659
+ spin.stop();
2660
+ if (status !== 0) { console.error(`sys1grep: --summarize=${opt.summarize}: ${summarizer[0]} exited with ${status ?? signal}`); return false; }
2661
+ if (template) writeAnswer(answer);
2406
2662
  } else {
2407
- const body = await res.json();
2408
- const content = body?.choices?.[0]?.message?.content;
2409
- if (typeof content !== 'string') { console.error(`sys1grep: --summarize=${opt.summarize}: no answer in the response`); summaryFailed = true; }
2410
- else if (template) writeAnswer(content);
2411
- else process.stdout.write(content.endsWith('\n') ? content : `${content}\n`);
2663
+ // An OpenAI-compatible server (#76): one request, no CLI. A cold local model can take a while, well past Jev's
2664
+ // 60s timeout.
2665
+ spin.set(`summarizing with ${opt.summarize}`);
2666
+ let res;
2667
+ try {
2668
+ res = await fetch(summarizer.url, {
2669
+ method: 'POST',
2670
+ headers: { 'content-type': 'application/json', ...(summarizer.key && { authorization: `Bearer ${summarizer.key}` }) },
2671
+ body: JSON.stringify({
2672
+ model: summarizer.model,
2673
+ messages: [{ role: 'system', content: summarizer.prompt }, { role: 'user', content: text }],
2674
+ stream: false,
2675
+ reasoning_effort: 'none',
2676
+ }),
2677
+ signal: AbortSignal.timeout(300_000),
2678
+ });
2679
+ } catch (e) { die(`--summarize=${opt.summarize}: ${e.cause?.message ?? e.message}`, false); }
2680
+ spin.stop();
2681
+ if (!res.ok) {
2682
+ console.error(`sys1grep: --summarize=${opt.summarize}: ${res.status}: ${(await res.text()).slice(0, 300).replace(/[\x00-\x1f\x7f-\x9f]/g, ' ')}`);
2683
+ return false;
2684
+ } else {
2685
+ const body = await res.json();
2686
+ const content = body?.choices?.[0]?.message?.content;
2687
+ if (typeof content !== 'string') { console.error(`sys1grep: --summarize=${opt.summarize}: no answer in the response`); return false; }
2688
+ else if (template) writeAnswer(content);
2689
+ else process.stdout.write(content.endsWith('\n') ? content : `${content}\n`);
2690
+ }
2412
2691
  }
2692
+ return true;
2413
2693
  }
2414
2694
  // Summary only when interactive; grep prints nothing to stderr when scripted.
2415
2695
  if (dry) {
2416
- const assumed = [willFold && `--dedup=${opt.dedup}`, opt.unit === 'sentence-by-jev' && '--unit=sentence-by-jev'].filter(Boolean);
2417
- const tokens = estimateTokens(traced, tracedBytes, tracedCjkBytes), price = customUrl ? '' : `, ~$${(tokens * 0.042 / 1e6).toFixed(6)}`;
2696
+ const assumed = [willFold && opt.dedup === 'always' && '--dedup=always', opt.unit === 'sentence-by-jev' && '--unit=sentence-by-jev'].filter(Boolean);
2697
+ // --dedup=auto asks no question; a dry run answers every sample no, so they always agree: the count is the floor
2698
+ const agreed = willFold && opt.dedup === 'auto' ? ' (--dedup=auto samples assumed to agree; a split sends the rest)' : '';
2699
+ const toSend = traced - cacheHits, tokens = estimateTokens(toSend, tracedBytes, tracedCjkBytes), price = `, ~$${(tokens * pricePerM / 1e6).toFixed(6)}${priceTag}`;
2418
2700
  // #125 review: the size guard is gone (an oversized file is skipped outright above, not asked about); --dry-run
2419
2701
  // and -i show only the cost guard's verdict here, consistent with what a real run would ask.
2420
- const guard = (tokens * 0.042) / 1e6 > MAX_COST ? `; over --max-cost ${MAX_COST}, would ask` : '';
2421
- logPlan(`dry run: ${traced} request${traced === 1 ? '' : 's'}, ${sent.length} of ${totalUnits} ${unitName} to send, ${tracedQuestions} questions, ${tracedChars} chars, ~${tokens} input tokens${price}; nothing sent${assumed.length ? ` (${assumed.join(' and ')} questions assumed no)` : ''}${guard}`);
2702
+ const guard = (tokens * pricePerM) / 1e6 > MAX_COST ? `; over --max-cost ${MAX_COST}, would ask` : '';
2703
+ const split = cacheMode === 'off' ? '' : ` to send, ${cacheHits} cached`;
2704
+ logPlan(`dry run: ${toSend} request${toSend === 1 ? '' : 's'}${split}, ${sent.length - cachedUnits} of ${totalUnits} ${unitName} to send, ${tracedQuestions} questions, ${tracedChars} chars, ~${tokens} input tokens${price}; nothing sent${assumed.length ? ` (${assumed.join(' and ')} questions assumed no)` : ''}${agreed}${guard}`);
2422
2705
  } else if ((process.stderr.isTTY || opt.verbose) && !opt.quiet) {
2423
2706
  // #143: dedup off by default until real-run stats say auto should be it; meanwhile a run that would have paid
2424
2707
  // says so, once, so a user stuck on the default default learns --dedup=auto exists.
2425
2708
  if (opt.dedup === 'never' && dedupEstimate?.pays) {
2426
2709
  console.error(`sys1grep: ${dedupEstimate.units} units fold to at most ${dedupEstimate.templates} templates; --dedup=auto would save ~${dedupEstimate.requests} requests (~${kify(dedupEstimate.saved)} tokens)`);
2427
2710
  }
2428
- // The API's own usage.cost when reported (OpenRouter does); else an estimate at Jev's list price, only for TypeSafe itself.
2429
- const perToken = usedCost > 0 && usedTokens > 0 ? usedCost / usedTokens : customUrl ? 0 : 0.042 / 1e6;
2430
- const cost = usedCost > 0 ? `, $${usedCost.toFixed(6)}` : perToken ? `, ~$${(usedTokens * perToken).toFixed(6)}` : '';
2711
+ // The API's own usage.cost when reported (OpenRouter does); else an estimate at pricePerM (see the block after `model`), tagged for another URL.
2712
+ const perToken = usedCost > 0 && usedTokens > 0 ? usedCost / usedTokens : pricePerM / 1e6;
2713
+ const cost = usedCost > 0 ? `, $${usedCost.toFixed(6)}` : usedTokens > 0 ? `, ~$${(usedTokens * perToken).toFixed(6)}${priceTag}` : '';
2431
2714
  // --dedup's savings: what the folded units would have cost as requests of their own (estimated, #92's fit), less
2432
2715
  // what its questions did cost (reported). A net figure: on prose, which barely folds, it can come out negative.
2433
2716
  let folded = '';
@@ -2438,10 +2721,38 @@ if (dry) {
2438
2721
  return t + estimateTokens(1, Buffer.byteLength(body), cjkBytesOf(body));
2439
2722
  }, 0) - dedupTokens;
2440
2723
  const share = saved + usedTokens > 0 ? `, ${Math.round((100 * saved) / (saved + usedTokens))}%` : '';
2441
- folded = ` (${members.length} folded by --dedup, ~${saved} input tokens${perToken ? ` / ~$${(saved * perToken).toFixed(6)}` : ''} saved${share})`;
2724
+ folded = ` (${members.length} folded by --dedup, ~${saved} input tokens / ~$${(saved * perToken).toFixed(6)} saved${share})`;
2442
2725
  }
2443
2726
  const n = (k, w) => `${k} ${w}${k === 1 ? '' : 's'}`;
2444
- console.error(`${matched} of ${totalUnits} ${unitName} matched; ${requestCount ? `${sent.length} sent to Jev${folded} in ${n(requestCount, 'request')}, ${n(usedTokens, 'input token')}${cost}` : 'nothing sent'}`);
2727
+ console.error(`${matched} of ${totalUnits} ${unitName} matched; ${requestCount ? `${sent.length - cachedUnits} sent to Jev${folded} in ${n(requestCount, 'request')}, ${n(usedTokens, 'input token')}${cost}` : 'nothing sent'}${cacheHits ? `, ${n(cacheHits, 'request')} answered from the cache` : ''}`);
2445
2728
  }
2446
2729
  // process.exit() can drop buffered stdout when piped, so set exitCode instead.
2447
- process.exitCode = dry ? (hadError ? 2 : 0) : matched && opt.quiet ? 0 : hadError || summaryFailed ? 2 : matched ? 0 : 1; // -q: a match wins over an error
2730
+ process.exitCode = dry ? (hadError ? 2 : 0) : matched && opt.quiet ? 0 : hadError || (summaryFailed && !askSummary) ? 2 : matched ? 0 : 1; // -q: a match wins over an error
2731
+ // #139: exit 1 is a threshold decision, not a fact; name the closest line so the caller does not rerun to see it.
2732
+ if (process.exitCode === 1 && sent.length && !opt.quiet && !multiStep) {
2733
+ // A term is as close as its lowest meaning: an AND holds only when every meaning in it clears the threshold.
2734
+ let best = { p: -1 };
2735
+ // the tNeg after following the advice: --level loose also loosens tNeg, a lower -t (or -T as given) does not
2736
+ const adviceNeg = opt.t !== undefined || opt.T !== undefined || tPos <= levels.loose[0] ? tNeg : levels.loose[1];
2737
+ for (const l of sent) for (const term of expr) {
2738
+ const { ok, matches } = regexPart(term, l);
2739
+ if (!ok) continue;
2740
+ // a negation that fails even after the advice holds the term down: that unit is no candidate (the termHolds test)
2741
+ if (term.some(lit => lit.kind === 'm' && lit.not && (asksByUnit.get(l).get(expandCaptures(lit.text, matches)) ?? 0) >= adviceNeg)) continue;
2742
+ let low = null;
2743
+ for (const lit of term) if (lit.kind === 'm' && !lit.not) {
2744
+ const p = asksByUnit.get(l).get(expandCaptures(lit.text, matches)) ?? 0;
2745
+ if (!low || p < low.p) low = { p, l, meaning: lit.said };
2746
+ }
2747
+ // only a term its own meanings failed: a near miss is by definition under tPos
2748
+ if (low && low.p < tPos && low.p > best.p) best = low;
2749
+ }
2750
+ if (best.l) {
2751
+ const at = `${best.l.file === '-' ? 'standard input' : best.l.file}:${spansOf.get(best.l.file)?.[best.l.no - 1]?.[0]?.[0] ?? best.l.no}`;
2752
+ // -t overrides --level, so with -t only a lower -t changes the outcome; advice that would not match is none
2753
+ const loose = tPos <= levels.loose[0] || best.p < levels.loose[0] ? '' : opt.t === undefined ? `--level loose takes ${levels.loose[0]}, ` : `-t ${levels.loose[0]} loosens it, `;
2754
+ // -p prints only what matches: -t 0 lets every unit match, -T 1 keeps a negation from dropping one
2755
+ const every = `${opt.p ? '' : '-p '}-t 0${expr.some(term => term.some(lit => lit.kind === 'm' && lit.not)) ? ' -T 1' : ''}`;
2756
+ console.error(`sys1grep: no ${unitName.slice(0, -1)} reached ${tPos} for "${safe(best.meaning)}"; the highest was ${(Math.floor(best.p * 100 + 1e-9) / 100).toFixed(2)} (${safe(at)}). ${loose}${every} shows every probability`);
2757
+ }
2758
+ }