@vibecompass/vibecompass 0.15.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +155 -0
- package/README.md +75 -0
- package/package.json +1 -1
- package/src/brief-keywords.js +261 -0
- package/src/brief-lifecycle.js +659 -0
- package/src/brief-render.js +352 -0
- package/src/brief-settings.js +160 -0
- package/src/brief.js +1714 -0
- package/src/cli.js +269 -3
- package/src/decision-lineage.js +1718 -0
- package/src/generators/agent-files/template.js +2 -1
- package/src/index.js +34 -0
- package/src/project-memory.js +30 -4
- package/src/read-model.js +92 -2
- package/src/scaffold.js +10 -2
- package/src/session.js +30 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,160 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.17.0 - 2026-10-03
|
|
4
|
+
|
|
5
|
+
Session brief retrieval and framing fixes (recall plan A7). The lane brief stays
|
|
6
|
+
opt-in (D-368). The re-evaluation meant to decide default-on stopped at a protocol
|
|
7
|
+
deviation before any run was graded, so it gives no evidence either way. These
|
|
8
|
+
fixes target misses from the earlier evaluation and were checked on its tuning
|
|
9
|
+
tasks with briefs alone, not in an agent evaluation.
|
|
10
|
+
|
|
11
|
+
**Breaking for `vibecompass brief --json` consumers:** contract version 2 removes
|
|
12
|
+
`selection.narrowed` (see below). Other commands and the read model are unchanged.
|
|
13
|
+
|
|
14
|
+
- Selection: a decision is also matched on the sentences in architecture docs
|
|
15
|
+
that cite it (the overview included; Review metadata and Involved files
|
|
16
|
+
excluded), so a doc's own words reach a decision whose entry uses different
|
|
17
|
+
ones. A doc shares the keyword relevance of the decisions it cites, so a doc
|
|
18
|
+
whose Description misses the task's words is still reached through the
|
|
19
|
+
decisions it describes. Common abbreviations meet their long forms
|
|
20
|
+
(organization/org, repository/repo, configuration/config, database/db,
|
|
21
|
+
environment/env, documentation/doc, authentication/auth).
|
|
22
|
+
- No-match: evaluative words ("clearer", "flaky", "noticeably") and
|
|
23
|
+
kind-of-work words ("investigate", "failure", "bug") never count as a task's
|
|
24
|
+
distinctive topic words. When the distinctive topic words are mostly unknown
|
|
25
|
+
to memory and only keywords matched, the status is `no-match` with up to four
|
|
26
|
+
nearby reads (paths that share other task words, never presented as
|
|
27
|
+
relevant). This replaces the narrow `partial` match.
|
|
28
|
+
- Framing: a brief with units says it is a starting point of cut excerpts that
|
|
29
|
+
can miss the doc a task depends on, and names up to six of the task's words
|
|
30
|
+
to search for beyond it (`selection.search_terms` in `--json`).
|
|
31
|
+
- Lane `brief.md` format 2: `exclude` and `repo_aliases` show at most four
|
|
32
|
+
entries, bound by `exclude_digest` and `repo_aliases_digest`, so staleness
|
|
33
|
+
checks stay complete as settings and repos grow; a header comment says the
|
|
34
|
+
token budget covers the body only. Format 1 briefs read as stale and are
|
|
35
|
+
regenerated on resume.
|
|
36
|
+
- `--json` contract version 2: `selection.narrowed` is replaced by
|
|
37
|
+
`selection.topic_absent` and `selection.search_terms`; follow-ups gain the
|
|
38
|
+
`nearby` priority.
|
|
39
|
+
|
|
40
|
+
## 0.16.0 - 2026-09-30
|
|
41
|
+
|
|
42
|
+
The session brief ships as an opt-in experiment (D-368). In a baseline-versus-brief
|
|
43
|
+
evaluation that did not fully follow its own protocol, the brief arm met the numerical
|
|
44
|
+
recall targets and made none of the labeled forbidden authority claims, but lost nine
|
|
45
|
+
facts that agents without a brief found. That does not support turning it on by
|
|
46
|
+
default, so `start-session` and `continue-session` write a lane brief only when
|
|
47
|
+
`project.yaml` sets `brief.enabled: true`. `vibecompass brief` works on request
|
|
48
|
+
either way.
|
|
49
|
+
|
|
50
|
+
- Read model: architecture components now carry `retrieval_guidance` (the
|
|
51
|
+
`## Retrieval guidance` section) and `retrieval_scope` (the "Retrieval
|
|
52
|
+
scope" bullet under `## Review metadata`); both are `null` when absent.
|
|
53
|
+
Existing fields are unchanged.
|
|
54
|
+
- Add typed decision references and declared decision lineage
|
|
55
|
+
(`decision_lineage` on the read model, contract version 1). Architecture
|
|
56
|
+
docs `cites` decisions, session notes `references` them and `made` the ones
|
|
57
|
+
listed at the start of a top-level "Decisions made" bullet, and decision
|
|
58
|
+
entries declare `supersedes` / `amends` / `preserves` only from their own
|
|
59
|
+
wording, with extent (`full` / `partial` / `scoped` / `unqualified`) and
|
|
60
|
+
scope. Other lineage-like wording is `unknown`; mentions are `references`;
|
|
61
|
+
nothing is inferred as "governs" or "currently valid" (D-359). Every
|
|
62
|
+
relation carries source path, section, line, document and section hashes,
|
|
63
|
+
and explicit-or-inferred evidence (range interiors are inferred).
|
|
64
|
+
- Lineage stays conservative: fenced examples, another decision's restated,
|
|
65
|
+
reported, or uncertain claim ("D-020 supersedes D-010", "As D-020
|
|
66
|
+
supersedes …", "Documents how D-020 supersedes …"), contrast objects
|
|
67
|
+
("separate from D-169"), hypothetical wording (may, would, if, unless, …),
|
|
68
|
+
and negated preservation ("cannot preserve") never certify a relation;
|
|
69
|
+
prohibited change ("cannot supersede", "cannot be superseded by <this
|
|
70
|
+
entry>") is preservation, and a requirement ("must retain") still
|
|
71
|
+
certifies preservation. An
|
|
72
|
+
`unknown` is kept beside certified relations unless one shares its object
|
|
73
|
+
(coordinated verbs). Historical retention ("kept for historical reference")
|
|
74
|
+
never overrides partial or scoped wording. Decision IDs must be positive
|
|
75
|
+
safe integers, so malformed ranges cannot loop.
|
|
76
|
+
- Recognize the optional structured lineage fields `**Supersedes:**`,
|
|
77
|
+
`**Partially supersedes:**`, `**Amends:**`, and `**Preserves:**` on new
|
|
78
|
+
decision entries (D-363); they win over prose for the targets they name, and
|
|
79
|
+
malformed items become diagnostics.
|
|
80
|
+
- Add `getDecisionLineage(readModel, id)` (declared successors transitively,
|
|
81
|
+
incoming and outgoing lineage, citations, and the notes that made it) and
|
|
82
|
+
export the pure extractors (`extractDecisionFileRelations`,
|
|
83
|
+
`extractSessionNoteRelations`, `extractArchitectureDocCitations`,
|
|
84
|
+
`scanDecisionReferences`, `buildDecisionLineageModel`) with shared fixtures
|
|
85
|
+
in `src/tests/fixtures/decision-lineage/` for the app port.
|
|
86
|
+
- Add `vibecompass brief` and the read-only session brief engine
|
|
87
|
+
(`buildSessionBrief`, `renderBrief`; plan task A3, D-359). Given a task, and
|
|
88
|
+
optionally files, feature slugs, claims, a lane, and a budget, it selects the
|
|
89
|
+
memory a session should read first — through the file-owner index (repo
|
|
90
|
+
aliases and workspace paths such as `vibecompass-app/src/…` both match),
|
|
91
|
+
lane-declared docs, feature slugs, and deterministic keyword matching on
|
|
92
|
+
titles, Description, and Retrieval guidance (no embeddings, no network, no
|
|
93
|
+
model) — and packs whole units (lane, decision-lineage, doc, note,
|
|
94
|
+
watch-out) into an estimated token budget (Unicode code points ÷ 4 over the
|
|
95
|
+
whole rendered brief; default 6,000, minimum 800). Every brief states
|
|
96
|
+
`complete`, `partial`, `incomplete` (with prioritized required reads, also
|
|
97
|
+
used for retrieval errors), or `no-match`, lists at most 12 follow-up reads
|
|
98
|
+
plus "+N more" (the full list is in `--json`), and never emits a decision
|
|
99
|
+
without every declared superseding or amending successor. Uncertified
|
|
100
|
+
lineage wording becomes a follow-up read, never a status. The command
|
|
101
|
+
writes nothing: no manifest refresh, state write, lane write, or agent-file
|
|
102
|
+
sync, so it is safe against a hand-maintained root.
|
|
103
|
+
- Session brief in the lane lifecycle (plan task A4, D-359, D-364, D-368),
|
|
104
|
+
opt-in: when `project.yaml` sets `brief.enabled: true`, `start-session` and
|
|
105
|
+
`continue-session` write `sessions/active/<lane-id>/brief.md` from the lane's working-on,
|
|
106
|
+
claims, and feature slugs, after every lifecycle write succeeded and inside
|
|
107
|
+
the same lock. `continue-session` keeps a current brief and regenerates a
|
|
108
|
+
missing or stale one, and lists it under "Read next". A brief failure is a
|
|
109
|
+
warning plus, when the file can be written, an `incomplete` brief naming the
|
|
110
|
+
reason; it never fails the command, and existing lifecycle refusals fail as
|
|
111
|
+
before. Without the setting (or with `brief.enabled: false`), lifecycle
|
|
112
|
+
commands print `Brief: off (…)`, write no brief, and leave a brief already
|
|
113
|
+
in the lane untouched; `--no-brief` (start and continue) skips generation
|
|
114
|
+
for one command on a root that opted in. Every other lifecycle effect is
|
|
115
|
+
unchanged. `vibecompass brief --write` writes or regenerates the lane brief
|
|
116
|
+
on request whatever the setting; `vibecompass brief --check [--json]`
|
|
117
|
+
reports whether it is stale, without writing.
|
|
118
|
+
- `brief.md` opens with a YAML header that binds its inputs: source root and
|
|
119
|
+
lane with the lane's session date and number, destination root and lane, a
|
|
120
|
+
digest of the canonical corpus it read, short hashes of the docs, decision
|
|
121
|
+
entries, and notes it drew text from, the lane scratch it read (`handoff.md`
|
|
122
|
+
bytes, the `session.yaml` fields it uses, the other active lanes), and the
|
|
123
|
+
repo alias map, plus ready-to-run check and refresh commands that keep the
|
|
124
|
+
root, lane, and source explicit. The body is exactly what
|
|
125
|
+
`vibecompass brief` prints for the lane. Staleness is computed from the
|
|
126
|
+
bindings, not stored: any hash, the session identity, or the alias map
|
|
127
|
+
differs, or the check cannot read what it needs; resume bookkeeping
|
|
128
|
+
(`resumed_at`, `resume_count`) never makes a brief stale. Memory that
|
|
129
|
+
cannot be read (invalid settings included) is a failed generation
|
|
130
|
+
(`generation: "failed"`; `brief --write` exits 1), while mandatory units
|
|
131
|
+
overflowing the budget is a successful `incomplete` brief.
|
|
132
|
+
- `project.yaml` gains an optional `brief:` mapping (D-364): `enabled`
|
|
133
|
+
(default `false`; `true` opts the root into lifecycle generation, D-368),
|
|
134
|
+
and `exclude`, a list of root-relative globs (`*`, `**`, `?`; case-insensitive)
|
|
135
|
+
naming canonical documents every brief must never read. Excluded files are
|
|
136
|
+
dropped from the scan before they are opened, so they contribute nothing to
|
|
137
|
+
selection, ranking, citation propagation, bindings, or output. Malformed or
|
|
138
|
+
unknown `brief` settings, or an unreadable `project.yaml`, fail closed: the
|
|
139
|
+
brief reads no canonical document and is `incomplete`; scans warn with
|
|
140
|
+
`project-brief-invalid`. `loadProjectReadModel` and other consumers are
|
|
141
|
+
unaffected.
|
|
142
|
+
- Dual-root source for dogfood roots (D-255, D-364): `continue-session`,
|
|
143
|
+
`brief --write`, and `brief --check` accept `--source-root <root>` (and
|
|
144
|
+
`--source-session <lane-id>`, which must equal the lane) to build the brief
|
|
145
|
+
from another memory root's matching lane — same id, session date, and
|
|
146
|
+
session number, or the brief is refused (`incomplete`, with the reason).
|
|
147
|
+
The source root is only read (no lock, manifest, lane, stamp, or agent-file
|
|
148
|
+
write) and is byte-identical afterwards; both roots' exclusions apply;
|
|
149
|
+
destination repo ids translate to the source's by matching remotes. The
|
|
150
|
+
option is never inferred: a brief built from another root is stale for a
|
|
151
|
+
run that does not name it, and that root is not read.
|
|
152
|
+
- Generated `context.md`, the managed agent block, and the scratch listings
|
|
153
|
+
now tell agents to read the lane's `brief.md`, when present, before planning,
|
|
154
|
+
and describe lifecycle generation as opt-in. The lane
|
|
155
|
+
`wip.md` template gains an optional `## Context used` section for the docs
|
|
156
|
+
and decisions the lane actually relied on.
|
|
157
|
+
|
|
3
158
|
## 0.15.0 - 2026-09-25
|
|
4
159
|
|
|
5
160
|
- Add a layered local credential store for hosted sync tokens (D-355). Every
|
package/README.md
CHANGED
|
@@ -143,6 +143,17 @@ npx -y @vibecompass/vibecompass@latest close-session --root .compass --session a
|
|
|
143
143
|
# start-session --id <existing-lane> fails closed and points at continue-session.
|
|
144
144
|
npx -y @vibecompass/vibecompass@latest continue-session auth-flow --root .compass --working-on "Auth flow, phase 2"
|
|
145
145
|
|
|
146
|
+
# Session brief in the lane (D-359, D-364), opt-in (D-368): with
|
|
147
|
+
# project.yaml `brief: { enabled: true }`, start-session and continue-session
|
|
148
|
+
# write sessions/active/<lane-id>/brief.md — the memory the lane's task should
|
|
149
|
+
# start from, with a YAML header binding the inputs it read. continue-session
|
|
150
|
+
# keeps a current brief and regenerates a stale one; a brief failure is a
|
|
151
|
+
# warning, never a failed start or resume. Without the setting nothing is
|
|
152
|
+
# written, and --no-brief skips it for one command. brief --write briefs any
|
|
153
|
+
# lane on request; `brief.exclude` lists root-relative globs no brief may read.
|
|
154
|
+
npx -y @vibecompass/vibecompass@latest brief --root .compass --session auth-flow --write # write or regenerate it
|
|
155
|
+
npx -y @vibecompass/vibecompass@latest brief --root .compass --session auth-flow --check # current or stale, and why
|
|
156
|
+
|
|
146
157
|
# Optional git binding for a lane (D-281): create/reuse a branch in each bound
|
|
147
158
|
# repo; --worktree provisions per-repo worktrees under
|
|
148
159
|
# <workspace>/worktrees/<lane-id>/<repo-id> with the lane marker in the
|
|
@@ -186,6 +197,12 @@ npx -y @vibecompass/vibecompass@latest sync-agents --root .compass
|
|
|
186
197
|
npx -y @vibecompass/vibecompass@latest append-decision --root .compass --target cross-cutting.md --entry staged-entry.md
|
|
187
198
|
npx -y @vibecompass/vibecompass@latest refresh-decision-index --root .compass --check
|
|
188
199
|
|
|
200
|
+
# Session brief (read-only): the memory a task should start from, packed into
|
|
201
|
+
# an estimated token budget, with a complete/partial/incomplete/no-match status
|
|
202
|
+
# and follow-up reads. Decisions always appear with their declared successors.
|
|
203
|
+
npx -y @vibecompass/vibecompass@latest brief --root .compass --task "Change the refresh denial message" --files app:src/lib/entitlements.ts
|
|
204
|
+
npx -y @vibecompass/vibecompass@latest brief --root .compass --session auth-flow --budget 3000 --json
|
|
205
|
+
|
|
189
206
|
# Targeted and comprehensive architecture docs maintenance
|
|
190
207
|
npx -y @vibecompass/vibecompass@latest docs-update --root .compass --session auth-flow --changed app:src/auth/login.ts
|
|
191
208
|
npx -y @vibecompass/vibecompass@latest docs-review --root .compass --guided
|
|
@@ -240,6 +257,42 @@ workflow files, updating handoffs, and applying reviewed changes. Use
|
|
|
240
257
|
`docs update` / `vibecompass docs-update` for ordinary session-delta
|
|
241
258
|
maintenance tied to the current lane.
|
|
242
259
|
|
|
260
|
+
## Session Brief (opt-in)
|
|
261
|
+
|
|
262
|
+
`vibecompass brief` picks the architecture docs, decisions, and session notes a
|
|
263
|
+
task should start from, and packs them into a short markdown brief with a
|
|
264
|
+
`complete`, `partial`, `incomplete`, or `no-match` status and a list of
|
|
265
|
+
follow-up reads. It runs locally and deterministically: no network, no model,
|
|
266
|
+
no account. A decision always appears together with any later decision whose
|
|
267
|
+
own text explicitly declares that it replaces or amends it, and the brief never
|
|
268
|
+
claims that a decision governs your task. It cannot spot a replacement that is
|
|
269
|
+
only implied, so read the follow-up list too. When the task's distinctive words
|
|
270
|
+
match nothing in memory, the status is `no-match`, with at most four nearby
|
|
271
|
+
reads that share the task's other words.
|
|
272
|
+
|
|
273
|
+
The brief in a session lane is an experiment you turn on (D-368):
|
|
274
|
+
|
|
275
|
+
```yaml
|
|
276
|
+
# .compass/project.yaml
|
|
277
|
+
brief:
|
|
278
|
+
enabled: true
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
With it on, `start-session` and `continue-session` write
|
|
282
|
+
`sessions/active/<lane-id>/brief.md`, and your AI tool reads it before
|
|
283
|
+
planning. Without it, nothing changes, and
|
|
284
|
+
`vibecompass brief --session <lane-id> --write` still briefs one lane on
|
|
285
|
+
request. `brief.exclude` lists files no brief may read.
|
|
286
|
+
|
|
287
|
+
The brief is opt-in because of the project's own evaluation, which did not
|
|
288
|
+
fully follow its protocol. Agents with a brief found about as much of the
|
|
289
|
+
relevant memory as agents without one, and made none of the specific false
|
|
290
|
+
authority claims the evaluation checked for. But they missed nine facts the
|
|
291
|
+
others found, in one case after the brief left out the key document. A second
|
|
292
|
+
evaluation, after retrieval fixes, stopped at a protocol deviation before any
|
|
293
|
+
grading, so turning the brief on by default still waits for a conforming one.
|
|
294
|
+
Treat a brief as a starting point, not the whole answer.
|
|
295
|
+
|
|
243
296
|
## Hosted Sync
|
|
244
297
|
|
|
245
298
|
Use hosted sync when you want browser review, team workflows, or hosted
|
|
@@ -336,6 +389,9 @@ import {
|
|
|
336
389
|
scanProjectMemory,
|
|
337
390
|
loadProjectReadModel,
|
|
338
391
|
getProjectContext,
|
|
392
|
+
getDecisionLineage,
|
|
393
|
+
buildSessionBrief,
|
|
394
|
+
renderBrief,
|
|
339
395
|
detectChangelogShapedLines,
|
|
340
396
|
ARCHITECTURE_DOC_SOFT_SIZE_LIMIT_BYTES,
|
|
341
397
|
} from "@vibecompass/vibecompass";
|
|
@@ -351,6 +407,25 @@ so external validators (for example the hosted proposal parser) stay in parity.
|
|
|
351
407
|
`ARCHITECTURE_DOC_SOFT_SIZE_LIMIT_BYTES` is the byte-valued soft size budget
|
|
352
408
|
(12000) both docs-review apply warnings and the docs-update size advisory read.
|
|
353
409
|
|
|
410
|
+
`loadProjectReadModel(root)` also returns `decision_lineage`: evidence-typed
|
|
411
|
+
relations between documents and decisions. Architecture docs `cites`
|
|
412
|
+
decisions; session notes `references` them and `made` the ones listed under
|
|
413
|
+
"Decisions made". Decision entries declare `supersedes`, `amends`, or
|
|
414
|
+
`preserves` only from their own wording, with an extent and scope; anything
|
|
415
|
+
the parser cannot certify is `unknown`. Every relation carries its source
|
|
416
|
+
path, section, line, and content hashes. `getDecisionLineage(readModel, id)`
|
|
417
|
+
returns one decision's incoming and outgoing lineage and its declared
|
|
418
|
+
successors, followed transitively. Nothing is ever reported as "governs" or
|
|
419
|
+
"currently valid". Architecture components also carry `retrieval_guidance`
|
|
420
|
+
and `retrieval_scope` when their docs define them.
|
|
421
|
+
|
|
422
|
+
`buildSessionBrief({ rootDir, task, files, featureSlugs, claims, laneId, budget })`
|
|
423
|
+
is the read-only engine behind `vibecompass brief`: it returns the selection
|
|
424
|
+
result (status, packed units, omitted units, follow-up reads, gaps, and the
|
|
425
|
+
estimated size), and `renderBrief(result)` turns it into markdown. It never
|
|
426
|
+
writes to the root, and it never reads a path the root's `project.yaml`
|
|
427
|
+
`brief.exclude` names.
|
|
428
|
+
|
|
354
429
|
## More Documentation
|
|
355
430
|
|
|
356
431
|
- Developer guide: https://vibecompass.dev/developers
|
package/package.json
CHANGED
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic keyword matching for the session brief (plan task A3; D-359):
|
|
3
|
+
* a tokenizer with light suffix stemming and a BM25F-style scorer over a
|
|
4
|
+
* small fielded index. No embeddings, no network, no model — the same inputs
|
|
5
|
+
* always produce the same terms and scores.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
const STOPWORDS = new Set(
|
|
9
|
+
(
|
|
10
|
+
'a about above after again against all also am an and any are as at be because been before being below between both but by ' +
|
|
11
|
+
'can cannot could did do does doing done down during each either etc even ever every few for from further had has have having ' +
|
|
12
|
+
'he her here hers him his how however i if in into is it its itself just me more most much my neither no nor not now of off on ' +
|
|
13
|
+
'once one only onto or other our ours out over own per same she should so some such than that the their theirs them then there ' +
|
|
14
|
+
'these they this those though through thus to too under until up upon us very via was we were what whatever when where whether ' +
|
|
15
|
+
'which while who whom whose why will with within without would yet you your yours'
|
|
16
|
+
).split(' '),
|
|
17
|
+
);
|
|
18
|
+
|
|
19
|
+
// Instruction framing that says what to do, not what the task is about.
|
|
20
|
+
// Dropped from query text only; indexed memory keeps every content word.
|
|
21
|
+
const QUERY_FRAMING = new Set(
|
|
22
|
+
(
|
|
23
|
+
'add answer anything change create edit everything afterward afterwards happen implement include including instead make ' +
|
|
24
|
+
'modify need please rewrite something thing things walk want write'
|
|
25
|
+
).split(' '),
|
|
26
|
+
);
|
|
27
|
+
|
|
28
|
+
const IRREGULAR = new Map([
|
|
29
|
+
['paid', 'pay'],
|
|
30
|
+
['made', 'make'],
|
|
31
|
+
['built', 'build'],
|
|
32
|
+
['ran', 'run'],
|
|
33
|
+
['written', 'write'],
|
|
34
|
+
['wrote', 'write'],
|
|
35
|
+
['chosen', 'choose'],
|
|
36
|
+
['chose', 'choose'],
|
|
37
|
+
]);
|
|
38
|
+
|
|
39
|
+
// Ordered suffix rules: the first rule whose suffix matches (with at least
|
|
40
|
+
// `min` characters left) applies. A final `e` is dropped afterwards so
|
|
41
|
+
// "translate" and "translation" meet at "translat".
|
|
42
|
+
const SUFFIX_RULES = [
|
|
43
|
+
['ational', 'ate', 3],
|
|
44
|
+
['izations', 'ize', 3],
|
|
45
|
+
['ization', 'ize', 3],
|
|
46
|
+
['ations', 'ate', 3],
|
|
47
|
+
['ation', 'ate', 3],
|
|
48
|
+
['ements', '', 4],
|
|
49
|
+
['ement', '', 4],
|
|
50
|
+
['ments', '', 3],
|
|
51
|
+
['ment', '', 3],
|
|
52
|
+
['nesses', '', 3],
|
|
53
|
+
['ness', '', 3],
|
|
54
|
+
['ingly', '', 3],
|
|
55
|
+
['ings', '', 3],
|
|
56
|
+
['ing', '', 3],
|
|
57
|
+
['edly', '', 3],
|
|
58
|
+
['ies', 'y', 2],
|
|
59
|
+
['ied', 'y', 2],
|
|
60
|
+
['sses', 'ss', 2],
|
|
61
|
+
['xes', 'x', 2],
|
|
62
|
+
['ches', 'ch', 2],
|
|
63
|
+
['shes', 'sh', 2],
|
|
64
|
+
['ed', '', 3],
|
|
65
|
+
['ly', '', 4],
|
|
66
|
+
['s', '', 3],
|
|
67
|
+
];
|
|
68
|
+
|
|
69
|
+
// Common abbreviations meet their long forms after stemming, at index and
|
|
70
|
+
// query time alike: "organizations" and "org" both become "org".
|
|
71
|
+
const CANONICAL_STEMS = new Map([
|
|
72
|
+
['organiz', 'org'],
|
|
73
|
+
['repository', 'repo'],
|
|
74
|
+
['configurat', 'config'],
|
|
75
|
+
['databas', 'db'],
|
|
76
|
+
['environ', 'env'],
|
|
77
|
+
['documentat', 'doc'],
|
|
78
|
+
['authenticat', 'auth'],
|
|
79
|
+
]);
|
|
80
|
+
|
|
81
|
+
export function stemWord(word) {
|
|
82
|
+
const stem = stemRaw(word);
|
|
83
|
+
return CANONICAL_STEMS.get(stem) ?? stem;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function stemRaw(word) {
|
|
87
|
+
const lower = word.toLowerCase();
|
|
88
|
+
if (IRREGULAR.has(lower)) return IRREGULAR.get(lower);
|
|
89
|
+
if (lower.length <= 3 || lower.endsWith('ss') || lower.endsWith('us') || lower.endsWith('is')) {
|
|
90
|
+
return lower;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
let stem = lower;
|
|
94
|
+
for (const [suffix, replacement, min] of SUFFIX_RULES) {
|
|
95
|
+
if (stem.endsWith(suffix) && stem.length - suffix.length >= min) {
|
|
96
|
+
stem = stem.slice(0, stem.length - suffix.length) + replacement;
|
|
97
|
+
break;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// "running" → "runn" → "run"; keep "ll"/"ss"/"zz" doubles ("install", "access").
|
|
102
|
+
if (stem !== lower && /([b-df-hj-kmnp-rtv-y])\1$/.test(stem) && !/(ll|ss|zz)$/.test(stem)) {
|
|
103
|
+
stem = stem.slice(0, -1);
|
|
104
|
+
}
|
|
105
|
+
if (stem.length > 4 && stem.endsWith('e')) {
|
|
106
|
+
stem = stem.slice(0, -1);
|
|
107
|
+
}
|
|
108
|
+
return stem;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// Words that say how a user feels about a topic ("clearer", "flaky",
|
|
112
|
+
// "noticeably", "honestly") or what kind of work they want ("investigate",
|
|
113
|
+
// "failure", "bug") rather than what the topic is. They still count for
|
|
114
|
+
// keyword scoring; they are never a task's distinctive topic words, so they
|
|
115
|
+
// alone cannot make a brief abstain.
|
|
116
|
+
const NON_TOPICAL_WORDS = new Set(
|
|
117
|
+
(
|
|
118
|
+
'actual annoying awful awkward bad basic beautiful better best big brittle broken buggy careful clean clear clever clumsy clunky ' +
|
|
119
|
+
'complete confus confusing constant curious decent difficult dumb easy elegant entire ever extreme fast feel fine flaky fragile frank ' +
|
|
120
|
+
'frequent friendly frustrating full general glitchy good great happy hard heavy helpful honest hopeful huge ideal important ' +
|
|
121
|
+
'intermittent intuitive janky laggy large light little lovely main maybe messy minor modern mostly much nice noisy noticeable obvious ' +
|
|
122
|
+
'occasional odd often overall painful perhaps pleasant polished poor possible pretty probable quick quiet quite random rare real ' +
|
|
123
|
+
'rather reliable responsive rough seem serious simple skeptical slight slow sluggish small smart smooth snappy solid somehow ' +
|
|
124
|
+
'sometime sometimes somewhat sporadic strange stupid sturdy subtle super terrible thorough tidy tiny total ugly unclear unfriendly ' +
|
|
125
|
+
'unhappy unpleasant unreliable unstable usable useful useless usual verbose very weird worried worse worst ' +
|
|
126
|
+
// Kinds of work and trouble.
|
|
127
|
+
'behave behavior behaviour bug check crash debug diagnose error explain fail failure figure fix handle help idea improve ' +
|
|
128
|
+
'improvement incorrect investigate issue missing problem question refactor regression repair tweak understand verify wrong'
|
|
129
|
+
).split(' '),
|
|
130
|
+
);
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* True for a word that only evaluates, hedges, intensifies, or names a kind of
|
|
134
|
+
* work (comparatives, -ly adverbs, and -s/-ed/-ing forms included).
|
|
135
|
+
*/
|
|
136
|
+
export function isNonTopicalWord(word) {
|
|
137
|
+
const lower = String(word).toLowerCase();
|
|
138
|
+
const forms = new Set([lower]);
|
|
139
|
+
for (const [suffix, replacement] of [
|
|
140
|
+
['iest', 'y'], ['ier', 'y'], ['est', ''], ['er', ''], ['ily', 'y'], ['ably', 'able'], ['ibly', 'ible'], ['ly', ''],
|
|
141
|
+
['ing', ''], ['ing', 'e'], ['ed', ''], ['ed', 'e'], ['es', ''], ['s', ''],
|
|
142
|
+
]) {
|
|
143
|
+
if (lower.endsWith(suffix) && lower.length - suffix.length >= 3) forms.add(lower.slice(0, lower.length - suffix.length) + replacement);
|
|
144
|
+
}
|
|
145
|
+
for (const form of [...forms]) {
|
|
146
|
+
if (/(.)\1$/.test(form)) forms.add(form.slice(0, -1)); // "bigger" → "bigg" → "big"
|
|
147
|
+
if (form.endsWith('i')) forms.add(`${form.slice(0, -1)}y`);
|
|
148
|
+
}
|
|
149
|
+
return [...forms].some((form) => NON_TOPICAL_WORDS.has(form));
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** Raw lowercase word tokens: letters and digits, split on everything else. */
|
|
153
|
+
export function splitWords(text) {
|
|
154
|
+
return String(text ?? '')
|
|
155
|
+
.toLowerCase()
|
|
156
|
+
.split(/[^a-z0-9]+/)
|
|
157
|
+
.filter((word) => word.length >= 2 && !/^\d+$/.test(word));
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** Content terms for indexing: stopwords dropped, stemmed, order kept. */
|
|
161
|
+
export function indexTerms(text) {
|
|
162
|
+
const terms = [];
|
|
163
|
+
for (const word of splitWords(text)) {
|
|
164
|
+
if (STOPWORDS.has(word)) continue;
|
|
165
|
+
terms.push(stemWord(word));
|
|
166
|
+
}
|
|
167
|
+
return terms;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Unique query terms: stopwords and instruction framing dropped, stemmed. */
|
|
171
|
+
export function queryTerms(text) {
|
|
172
|
+
const seen = new Set();
|
|
173
|
+
const terms = [];
|
|
174
|
+
for (const word of splitWords(text)) {
|
|
175
|
+
if (STOPWORDS.has(word) || QUERY_FRAMING.has(word)) continue;
|
|
176
|
+
const stem = stemWord(word);
|
|
177
|
+
if (QUERY_FRAMING.has(stem) || seen.has(stem)) continue;
|
|
178
|
+
seen.add(stem);
|
|
179
|
+
terms.push(stem);
|
|
180
|
+
}
|
|
181
|
+
return terms;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** The first raw word behind each query term, for disclosures ("japanes" → "Japanese"). */
|
|
185
|
+
export function queryTermOrigins(text) {
|
|
186
|
+
const origins = new Map();
|
|
187
|
+
for (const word of String(text ?? '').split(/[^A-Za-z0-9]+/)) {
|
|
188
|
+
if (word.length < 2 || /^\d+$/.test(word)) continue;
|
|
189
|
+
const stem = stemWord(word.toLowerCase());
|
|
190
|
+
if (!origins.has(stem)) origins.set(stem, word);
|
|
191
|
+
}
|
|
192
|
+
return origins;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
const K1 = 1.2;
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* A fielded keyword index. `fields` maps a field name to `{ weight, b }`;
|
|
199
|
+
* each document supplies text per field.
|
|
200
|
+
*/
|
|
201
|
+
export function createKeywordIndex(fields, documents) {
|
|
202
|
+
const entries = documents.map((document) => {
|
|
203
|
+
const perField = {};
|
|
204
|
+
for (const name of Object.keys(fields)) {
|
|
205
|
+
const terms = indexTerms(document.fields[name] ?? '');
|
|
206
|
+
const counts = new Map();
|
|
207
|
+
for (const term of terms) counts.set(term, (counts.get(term) ?? 0) + 1);
|
|
208
|
+
perField[name] = { counts, length: terms.length };
|
|
209
|
+
}
|
|
210
|
+
return { id: document.id, perField };
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
const averageLength = {};
|
|
214
|
+
for (const name of Object.keys(fields)) {
|
|
215
|
+
const total = entries.reduce((sum, entry) => sum + entry.perField[name].length, 0);
|
|
216
|
+
averageLength[name] = entries.length > 0 ? Math.max(1, total / entries.length) : 1;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
const documentFrequency = new Map();
|
|
220
|
+
for (const entry of entries) {
|
|
221
|
+
const seen = new Set();
|
|
222
|
+
for (const name of Object.keys(fields)) {
|
|
223
|
+
for (const term of entry.perField[name].counts.keys()) seen.add(term);
|
|
224
|
+
}
|
|
225
|
+
for (const term of seen) documentFrequency.set(term, (documentFrequency.get(term) ?? 0) + 1);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
const count = entries.length;
|
|
229
|
+
const idf = (term) => {
|
|
230
|
+
const frequency = documentFrequency.get(term) ?? 0;
|
|
231
|
+
return Math.log(1 + (count - frequency + 0.5) / (frequency + 0.5));
|
|
232
|
+
};
|
|
233
|
+
|
|
234
|
+
return {
|
|
235
|
+
count,
|
|
236
|
+
documentFrequency: (term) => documentFrequency.get(term) ?? 0,
|
|
237
|
+
idf,
|
|
238
|
+
/** `{ id, score, matched }` per document with any matched term. */
|
|
239
|
+
score(terms) {
|
|
240
|
+
const results = [];
|
|
241
|
+
for (const entry of entries) {
|
|
242
|
+
let score = 0;
|
|
243
|
+
const matched = [];
|
|
244
|
+
for (const term of terms) {
|
|
245
|
+
let pseudo = 0;
|
|
246
|
+
for (const [name, { weight, b }] of Object.entries(fields)) {
|
|
247
|
+
const field = entry.perField[name];
|
|
248
|
+
const frequency = field.counts.get(term);
|
|
249
|
+
if (!frequency) continue;
|
|
250
|
+
pseudo += (weight * frequency) / (1 - b + (b * field.length) / averageLength[name]);
|
|
251
|
+
}
|
|
252
|
+
if (pseudo === 0) continue;
|
|
253
|
+
matched.push(term);
|
|
254
|
+
score += idf(term) * ((pseudo * (K1 + 1)) / (K1 + pseudo));
|
|
255
|
+
}
|
|
256
|
+
if (matched.length > 0) results.push({ id: entry.id, score, matched });
|
|
257
|
+
}
|
|
258
|
+
return results;
|
|
259
|
+
},
|
|
260
|
+
};
|
|
261
|
+
}
|