dsh-continual-evolve 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -6
- package/README.zh.md +12 -6
- package/lib/apply.d.ts +0 -2
- package/lib/apply.js +18 -8
- package/lib/auto.d.ts +8 -6
- package/lib/auto.js +42 -26
- package/lib/autocase.d.ts +38 -0
- package/lib/autocase.js +76 -0
- package/lib/benchmark-command.js +25 -1
- package/lib/benchmark.d.ts +0 -5
- package/lib/benchmark.js +1 -24
- package/lib/command.d.ts +2 -0
- package/lib/command.js +41 -5
- package/lib/consolidate.d.ts +90 -0
- package/lib/consolidate.js +180 -0
- package/lib/failures.d.ts +4 -1
- package/lib/failures.js +2 -1
- package/lib/index.d.ts +12 -0
- package/lib/index.js +11 -1
- package/lib/inject.d.ts +13 -17
- package/lib/inject.js +49 -37
- package/lib/mount.js +9 -0
- package/lib/promotion.d.ts +38 -6
- package/lib/promotion.js +93 -24
- package/lib/rollback.js +7 -0
- package/lib/search.d.ts +83 -0
- package/lib/search.js +136 -0
- package/lib/service.js +74 -1
- package/lib/state.d.ts +0 -12
- package/lib/state.js +4 -19
- package/lib/store.js +3 -1
- package/lib/tool.js +8 -1
- package/lib/types.d.ts +22 -0
- package/lib/types.js +22 -0
- package/lib/usage.d.ts +1 -10
- package/lib/usage.js +0 -17
- package/lib/validate.d.ts +10 -3
- package/lib/validate.js +52 -5
- package/lib/wrapup-command.js +17 -1
- package/lib/wrapup.d.ts +25 -1
- package/lib/wrapup.js +57 -6
- package/package.json +1 -1
package/lib/promotion.d.ts
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
* Pure functions only — the callers (wrapup command, gate local-fate phase)
|
|
15
15
|
* supply the resolved {@link PromotionPolicy}.
|
|
16
16
|
*/
|
|
17
|
-
import type { HarnessState, RefinementKind } from "./types.js";
|
|
17
|
+
import type { HarnessEntry, HarnessState, RefinementKind } from "./types.js";
|
|
18
18
|
export interface PromotionPolicy {
|
|
19
19
|
/** Content matching any pattern is project-scoped and stays local. */
|
|
20
20
|
blockPatterns: RegExp[];
|
|
@@ -24,6 +24,22 @@ export interface PromotionPolicy {
|
|
|
24
24
|
maxContentOverlap: number;
|
|
25
25
|
}
|
|
26
26
|
export declare const DEFAULT_PROMOTION_POLICY: PromotionPolicy;
|
|
27
|
+
/**
|
|
28
|
+
* Write-time conflict guard (R2): a global create whose similarity against an
|
|
29
|
+
* existing same-kind entry reaches this score is rejected outright — a
|
|
30
|
+
* near-duplicate adds zero information and the model should evolve_update
|
|
31
|
+
* the existing entry instead.
|
|
32
|
+
*/
|
|
33
|
+
export declare const CONFLICT_BLOCK_SCORE = 0.8;
|
|
34
|
+
/** Similarity at/above this stamps {@link CONFLICT_HINT_KEY} but lets the write proceed. */
|
|
35
|
+
export declare const CONFLICT_WARN_SCORE = 0.5;
|
|
36
|
+
/**
|
|
37
|
+
* First reason the content reads as carrying a live credential, or undefined
|
|
38
|
+
* when it screens clean. Unlike {@link projectScopedReason} this guard is
|
|
39
|
+
* NOT policy-configurable — it protects every sink at once (promotion path,
|
|
40
|
+
* global writes at the engine throat, mount materialization).
|
|
41
|
+
*/
|
|
42
|
+
export declare function secretLeakReason(content: string): string | undefined;
|
|
27
43
|
/**
|
|
28
44
|
* Build a policy from config values (schemastery strings compiled here so
|
|
29
45
|
* the config layer never touches RegExp). Invalid patterns are skipped —
|
|
@@ -41,9 +57,13 @@ export declare function resolvePromotionPolicy(options: {
|
|
|
41
57
|
*/
|
|
42
58
|
export declare function projectScopedReason(content: string, policy: PromotionPolicy): string | undefined;
|
|
43
59
|
/**
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
60
|
+
* Token set for cheap similarity, built on the search module's tokenizer —
|
|
61
|
+
* one home for "how this project splits words" (ASCII words lowercased plus
|
|
62
|
+
* CJK character bigrams; bigrams survive segmentation-free Chinese text).
|
|
63
|
+
* The set form feeds Jaccard overlap (review audit 2026-08-28 C: the
|
|
64
|
+
* previous private tokenizer duplicated the definition; the calibrated
|
|
65
|
+
* 0.6/0.8 thresholds are judgment values, not data-fitted, so the merge
|
|
66
|
+
* costs at most a test re-pin).
|
|
47
67
|
*/
|
|
48
68
|
export declare function normalizedTokens(text: string): Set<string>;
|
|
49
69
|
/** Jaccard similarity of two texts' normalized token sets (0..1). */
|
|
@@ -53,10 +73,22 @@ export interface SimilarEntryHit {
|
|
|
53
73
|
title: string;
|
|
54
74
|
score: number;
|
|
55
75
|
}
|
|
76
|
+
/**
|
|
77
|
+
* Human/LLM-readable description of a similarity hit, shared by the block
|
|
78
|
+
* error and the approval-question suffix so both surfaces explain the same
|
|
79
|
+
* way.
|
|
80
|
+
*/
|
|
81
|
+
export declare function buildConflictNotice(hit: SimilarEntryHit): string;
|
|
82
|
+
/**
|
|
83
|
+
* The most similar entry of the list, above `minScore`. Title and content
|
|
84
|
+
* both feed the comparison (titles are short; content carries the real
|
|
85
|
+
* signal). Generic form of {@link mostSimilarGlobalEntry} — callers decide
|
|
86
|
+
* which corpus (global store, merged view) the candidates come from.
|
|
87
|
+
*/
|
|
88
|
+
export declare function mostSimilarEntry(entries: readonly HarnessEntry[], title: string, content: string, minScore: number): SimilarEntryHit | undefined;
|
|
56
89
|
/**
|
|
57
90
|
* The most similar non-archived global entry of the same kind, above the
|
|
58
|
-
* policy threshold.
|
|
59
|
-
* short; content carries the real signal).
|
|
91
|
+
* policy threshold.
|
|
60
92
|
*/
|
|
61
93
|
export declare function mostSimilarGlobalEntry(globalState: HarnessState, kind: RefinementKind, title: string, content: string, policy: PromotionPolicy): SimilarEntryHit | undefined;
|
|
62
94
|
//# sourceMappingURL=promotion.d.ts.map
|
package/lib/promotion.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { isArchived } from "./types.js";
|
|
2
|
+
import { tokenize } from "./search.js";
|
|
2
3
|
/** Regex sources that mark content as project-scoped (never global). */
|
|
3
4
|
const DEFAULT_BLOCK_PATTERNS = [
|
|
4
5
|
String.raw `\/(?:mnt|home|Users)\/`, // absolute POSIX paths: "/home/…", "/mnt/…", "/Users/…"
|
|
@@ -10,6 +11,68 @@ export const DEFAULT_PROMOTION_POLICY = {
|
|
|
10
11
|
minPromoteChars: 100,
|
|
11
12
|
maxContentOverlap: 0.6,
|
|
12
13
|
};
|
|
14
|
+
/**
|
|
15
|
+
* Write-time conflict guard (R2): a global create whose similarity against an
|
|
16
|
+
* existing same-kind entry reaches this score is rejected outright — a
|
|
17
|
+
* near-duplicate adds zero information and the model should evolve_update
|
|
18
|
+
* the existing entry instead.
|
|
19
|
+
*/
|
|
20
|
+
export const CONFLICT_BLOCK_SCORE = 0.8;
|
|
21
|
+
/** Similarity at/above this stamps {@link CONFLICT_HINT_KEY} but lets the write proceed. */
|
|
22
|
+
export const CONFLICT_WARN_SCORE = 0.5;
|
|
23
|
+
/**
|
|
24
|
+
* Fixed secret-detection patterns — a security invariant, deliberately NOT
|
|
25
|
+
* part of the configurable {@link PromotionPolicy}: a user pattern typo must
|
|
26
|
+
* never be able to disable secret screening. Each entry pairs a human label
|
|
27
|
+
* with a regex tuned for low false positives (placeholders like
|
|
28
|
+
* "YOUR_API_KEY_HERE" don't match; realistic mixed literals do).
|
|
29
|
+
*/
|
|
30
|
+
const SECRET_PATTERNS = [
|
|
31
|
+
{ label: "AnySearch API key", regex: /\bas_sk_[A-Za-z0-9]{8,}\b/ },
|
|
32
|
+
{ label: "Anthropic API key", regex: /\bsk-ant-[A-Za-z0-9_-]{16,}\b/ },
|
|
33
|
+
{ label: "OpenAI-style API key", regex: /\bsk-(?:proj-)?[A-Za-z0-9_-]{16,}\b/ },
|
|
34
|
+
{ label: "GitHub token", regex: /\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{20,}\b/ },
|
|
35
|
+
{ label: "GitHub fine-grained PAT", regex: /\bgithub_pat_[A-Za-z0-9_]{20,}\b/ },
|
|
36
|
+
{ label: "AWS access key", regex: /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/ },
|
|
37
|
+
{ label: "Google API key", regex: /\bAIza[0-9A-Za-z_-]{35}\b/ },
|
|
38
|
+
{ label: "Slack token", regex: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/ },
|
|
39
|
+
{ label: "npm grant token", regex: /\bnpm_[A-Za-z0-9]{36}\b/ },
|
|
40
|
+
{ label: "private key block", regex: /-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY(?: BLOCK)?-----/ },
|
|
41
|
+
{
|
|
42
|
+
label: "credential assignment",
|
|
43
|
+
// The credential-name group is manually case-insensitive ("apiKey" in
|
|
44
|
+
// JSON, "API_KEY" in env examples) so the placeholder lookahead stays
|
|
45
|
+
// case-sensitive: it excludes only ALL-CAPS placeholders
|
|
46
|
+
// ("YOUR_KEY_HERE"), not real mixed/lowercase literals.
|
|
47
|
+
regex: /\b(?:[Aa][Pp][Ii][_-]?[Kk][Ee][Yy]|[Ss][Ee][Cc][Rr][Ee][Tt]|[Tt][Oo][Kk][Ee][Nn]|[Pp][Aa][Ss][Ss][Ww][Oo][Rr][Dd]|[Pp][Aa][Ss][Ss][Ww][Dd]|[Pp][Ww][Dd])\b["']?\s*[:=]\s*["'](?!["']*[A-Z0-9_]+["'])[A-Za-z0-9+/_-]{16,}["']/,
|
|
48
|
+
},
|
|
49
|
+
];
|
|
50
|
+
/**
|
|
51
|
+
* Redact a matched secret for error/audit surfaces: enough to identify the
|
|
52
|
+
* credential family, never the full value (reasons land in reviews.jsonl,
|
|
53
|
+
* question texts, and logs).
|
|
54
|
+
*/
|
|
55
|
+
function redactSecret(matched) {
|
|
56
|
+
if (matched.length <= 10) {
|
|
57
|
+
return `${matched.slice(0, 3)}…`;
|
|
58
|
+
}
|
|
59
|
+
return `${matched.slice(0, 6)}…${matched.slice(-3)}`;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* First reason the content reads as carrying a live credential, or undefined
|
|
63
|
+
* when it screens clean. Unlike {@link projectScopedReason} this guard is
|
|
64
|
+
* NOT policy-configurable — it protects every sink at once (promotion path,
|
|
65
|
+
* global writes at the engine throat, mount materialization).
|
|
66
|
+
*/
|
|
67
|
+
export function secretLeakReason(content) {
|
|
68
|
+
for (const { label, regex } of SECRET_PATTERNS) {
|
|
69
|
+
const match = regex.exec(content);
|
|
70
|
+
if (match?.[0]) {
|
|
71
|
+
return `possible ${label} ("${redactSecret(match[0])}") — secrets must never be sedimented into harness state; rotate the credential and keep it out of the store`;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return undefined;
|
|
75
|
+
}
|
|
13
76
|
/**
|
|
14
77
|
* Build a policy from config values (schemastery strings compiled here so
|
|
15
78
|
* the config layer never touches RegExp). Invalid patterns are skipped —
|
|
@@ -46,26 +109,16 @@ export function projectScopedReason(content, policy) {
|
|
|
46
109
|
return undefined;
|
|
47
110
|
}
|
|
48
111
|
/**
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
112
|
+
* Token set for cheap similarity, built on the search module's tokenizer —
|
|
113
|
+
* one home for "how this project splits words" (ASCII words lowercased plus
|
|
114
|
+
* CJK character bigrams; bigrams survive segmentation-free Chinese text).
|
|
115
|
+
* The set form feeds Jaccard overlap (review audit 2026-08-28 C: the
|
|
116
|
+
* previous private tokenizer duplicated the definition; the calibrated
|
|
117
|
+
* 0.6/0.8 thresholds are judgment values, not data-fitted, so the merge
|
|
118
|
+
* costs at most a test re-pin).
|
|
52
119
|
*/
|
|
53
120
|
export function normalizedTokens(text) {
|
|
54
|
-
|
|
55
|
-
const tokens = new Set();
|
|
56
|
-
for (const match of lowered.matchAll(/[a-z0-9_]{2,}/g)) {
|
|
57
|
-
tokens.add(match[0] ?? "");
|
|
58
|
-
}
|
|
59
|
-
let previous;
|
|
60
|
-
for (const match of lowered.matchAll(/[\u3400-\u9fff]/g)) {
|
|
61
|
-
const char = match[0] ?? "";
|
|
62
|
-
if (previous !== undefined) {
|
|
63
|
-
tokens.add(`${previous}${char}`);
|
|
64
|
-
}
|
|
65
|
-
previous = char;
|
|
66
|
-
}
|
|
67
|
-
tokens.delete("");
|
|
68
|
-
return tokens;
|
|
121
|
+
return new Set(tokenize(text));
|
|
69
122
|
}
|
|
70
123
|
/** Jaccard similarity of two texts' normalized token sets (0..1). */
|
|
71
124
|
export function contentOverlap(a, b) {
|
|
@@ -83,20 +136,36 @@ export function contentOverlap(a, b) {
|
|
|
83
136
|
return intersection / (left.size + right.size - intersection);
|
|
84
137
|
}
|
|
85
138
|
/**
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
-
*
|
|
139
|
+
* Human/LLM-readable description of a similarity hit, shared by the block
|
|
140
|
+
* error and the approval-question suffix so both surfaces explain the same
|
|
141
|
+
* way.
|
|
89
142
|
*/
|
|
90
|
-
export function
|
|
143
|
+
export function buildConflictNotice(hit) {
|
|
144
|
+
return `near-duplicate of ${hit.id} 「${hit.title}」 (similarity ${Math.round(hit.score * 100)}%)`;
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* The most similar entry of the list, above `minScore`. Title and content
|
|
148
|
+
* both feed the comparison (titles are short; content carries the real
|
|
149
|
+
* signal). Generic form of {@link mostSimilarGlobalEntry} — callers decide
|
|
150
|
+
* which corpus (global store, merged view) the candidates come from.
|
|
151
|
+
*/
|
|
152
|
+
export function mostSimilarEntry(entries, title, content, minScore) {
|
|
91
153
|
let best;
|
|
92
|
-
for (const other of
|
|
154
|
+
for (const other of entries) {
|
|
93
155
|
if (isArchived(other))
|
|
94
156
|
continue;
|
|
95
157
|
const score = Math.max(contentOverlap(title, other.title), contentOverlap(content, other.content));
|
|
96
|
-
if (score >=
|
|
158
|
+
if (score >= minScore && (best === undefined || score > best.score)) {
|
|
97
159
|
best = { id: other.id, title: other.title, score };
|
|
98
160
|
}
|
|
99
161
|
}
|
|
100
162
|
return best;
|
|
101
163
|
}
|
|
164
|
+
/**
|
|
165
|
+
* The most similar non-archived global entry of the same kind, above the
|
|
166
|
+
* policy threshold.
|
|
167
|
+
*/
|
|
168
|
+
export function mostSimilarGlobalEntry(globalState, kind, title, content, policy) {
|
|
169
|
+
return mostSimilarEntry(Object.values(globalState.entries[kind]), title, content, policy.maxContentOverlap);
|
|
170
|
+
}
|
|
102
171
|
//# sourceMappingURL=promotion.js.map
|
package/lib/rollback.js
CHANGED
|
@@ -19,6 +19,8 @@ export function rollbackProposal(target) {
|
|
|
19
19
|
function inverseEdit(edit, refinementId) {
|
|
20
20
|
if (edit.before && edit.after) {
|
|
21
21
|
// Forward action was an update: restore the before snapshot.
|
|
22
|
+
// skill_kind rides along (review audit 2026-08-28 B4): without it a
|
|
23
|
+
// rollback could flip the skill form or fail validation on re-apply.
|
|
22
24
|
return {
|
|
23
25
|
action: "update",
|
|
24
26
|
kind: edit.kind,
|
|
@@ -28,12 +30,16 @@ function inverseEdit(edit, refinementId) {
|
|
|
28
30
|
path: edit.before.path,
|
|
29
31
|
reference: edit.before.reference,
|
|
30
32
|
arguments: edit.before.arguments,
|
|
33
|
+
...(edit.before.skill_kind !== undefined ? { skill_kind: edit.before.skill_kind } : {}),
|
|
31
34
|
metadata: edit.before.metadata,
|
|
32
35
|
reason: `Rollback ${refinementId}`,
|
|
33
36
|
};
|
|
34
37
|
}
|
|
35
38
|
if (edit.before) {
|
|
36
39
|
// Forward action was a delete: re-create the entry from the snapshot.
|
|
40
|
+
// skill_kind is required for the re-created skill to pass the same
|
|
41
|
+
// contract validation as the original create (B4: a deleted guidance
|
|
42
|
+
// skill was previously unrecoverable via rollback).
|
|
37
43
|
return {
|
|
38
44
|
action: "create",
|
|
39
45
|
kind: edit.kind,
|
|
@@ -43,6 +49,7 @@ function inverseEdit(edit, refinementId) {
|
|
|
43
49
|
path: edit.before.path,
|
|
44
50
|
reference: edit.before.reference,
|
|
45
51
|
arguments: edit.before.arguments,
|
|
52
|
+
...(edit.before.skill_kind !== undefined ? { skill_kind: edit.before.skill_kind } : {}),
|
|
46
53
|
metadata: edit.before.metadata,
|
|
47
54
|
reason: `Rollback ${refinementId}`,
|
|
48
55
|
};
|
package/lib/search.d.ts
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retrieval primitives for injection ranking: CJK-aware tokenization plus a
|
|
3
|
+
* dependency-free, field-weighted BM25 scorer (ADR
|
|
4
|
+
* `implemented/feature/2026-08-24-cjk-bigram-bm25-retrieval.md`).
|
|
5
|
+
*
|
|
6
|
+
* Why this shape:
|
|
7
|
+
* - the old scorer tokenized whole CJK runs as single tokens compared for
|
|
8
|
+
* exact equality, so any rewording of a Chinese query missed every entry
|
|
9
|
+
* (「检索升级」 never matched 「全文检索方案」); overlapping character
|
|
10
|
+
* bigrams — the standard cjk_bigram analyzer shape (Elasticsearch/
|
|
11
|
+
* OpenSearch) — fix recall while keeping English behavior identical;
|
|
12
|
+
* - raw hit counts weighted common words and distinctive terms alike; BM25's
|
|
13
|
+
* IDF separates them. We use Lucene's non-negative variant
|
|
14
|
+
* `ln(1 + (N - df + 0.5) / (df + 0.5))` so EVERY matched token scores > 0
|
|
15
|
+
* and unmatched documents score exactly 0, preserving rankEntries'
|
|
16
|
+
* "relevant outranks recent-but-irrelevant" ordering invariant;
|
|
17
|
+
* - the index lifetime is one ranking call. Stores hold tens~hundreds of
|
|
18
|
+
* entries (measured full rebuild + score: single-digit milliseconds), the
|
|
19
|
+
* JSON state files stay the single source of truth, and npm installers
|
|
20
|
+
* carry zero native/WASM baggage — better-sqlite3 FTS5, sql.js, MiniSearch,
|
|
21
|
+
* FlexSearch were all evaluated and rejected (see the ADR's alternatives).
|
|
22
|
+
*/
|
|
23
|
+
import type { HarnessEntry } from "./types.js";
|
|
24
|
+
/** BM25 term-saturation constant (Okapi k1). */
|
|
25
|
+
export declare const BM25_K1 = 1.5;
|
|
26
|
+
/** BM25 document-length normalization constant (Okapi b). */
|
|
27
|
+
export declare const BM25_B = 0.75;
|
|
28
|
+
/** Multiplier for title-field matches (title hits outweigh body hits 2:1). */
|
|
29
|
+
export declare const TITLE_FIELD_WEIGHT = 2;
|
|
30
|
+
/**
|
|
31
|
+
* Tokenize text for relevance scoring: lowercase ASCII runs become word
|
|
32
|
+
* tokens; CJK runs become overlapping character bigrams (a single-character
|
|
33
|
+
* run stays a unigram). Deterministic and allocation-light — safe to call per
|
|
34
|
+
* injection build.
|
|
35
|
+
*
|
|
36
|
+
* @param text Arbitrary entry or query text.
|
|
37
|
+
* @returns Tokens in occurrence order (duplicates kept; consumers dedupe when
|
|
38
|
+
* order-independent weighting is wanted).
|
|
39
|
+
*/
|
|
40
|
+
export declare function tokenize(text: string): string[];
|
|
41
|
+
/** Per-field corpus statistics for one ranking pass. */
|
|
42
|
+
interface FieldStats {
|
|
43
|
+
/** Term-frequency map per document, positioned like {@link RelevanceIndex.positions}. */
|
|
44
|
+
readonly tfs: ReadonlyArray<ReadonlyMap<string, number>>;
|
|
45
|
+
/** Token count per document (this field's dl for BM25 length normalization). */
|
|
46
|
+
readonly lengths: ReadonlyArray<number>;
|
|
47
|
+
/** Document frequency per distinct term across the corpus. */
|
|
48
|
+
readonly df: ReadonlyMap<string, number>;
|
|
49
|
+
/** Average token count across the corpus (0 when empty). */
|
|
50
|
+
readonly averageLength: number;
|
|
51
|
+
}
|
|
52
|
+
/** Precomputed corpus statistics backing {@link relevanceScore}. */
|
|
53
|
+
export interface RelevanceIndex {
|
|
54
|
+
/** Number of documents in the corpus. */
|
|
55
|
+
readonly size: number;
|
|
56
|
+
/** Identity positions: the exact entry objects the index was built from. */
|
|
57
|
+
readonly positions: ReadonlyMap<object, number>;
|
|
58
|
+
readonly titles: FieldStats;
|
|
59
|
+
readonly bodies: FieldStats;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Build the corpus statistics (per-field tf, df, average length) for one
|
|
63
|
+
* ranking call. Pure: the index reads its input snapshot and never mutates
|
|
64
|
+
* the entries.
|
|
65
|
+
*
|
|
66
|
+
* @param entries The candidate entries being ranked this call.
|
|
67
|
+
* @returns An index that answers {@link relevanceScore} for exactly these
|
|
68
|
+
* entries (identity-keyed).
|
|
69
|
+
*/
|
|
70
|
+
export declare function buildRelevanceIndex(entries: readonly HarnessEntry[]): RelevanceIndex;
|
|
71
|
+
/**
|
|
72
|
+
* Field-weighted BM25 score of `entry` against `query` within `index`:
|
|
73
|
+
* `TITLE_FIELD_WEIGHT × bm25(title) + bm25(body)`. Query tokens are deduped,
|
|
74
|
+
* so repeating a word adds no weight.
|
|
75
|
+
*
|
|
76
|
+
* @returns A score > 0 when at least one query token occurs in the entry
|
|
77
|
+
* (either field), and exactly 0 otherwise — the property rankEntries'
|
|
78
|
+
* relevance-first ordering relies on. Never throws; an entry outside
|
|
79
|
+
* the index simply scores 0.
|
|
80
|
+
*/
|
|
81
|
+
export declare function relevanceScore(index: RelevanceIndex, entry: HarnessEntry, query: string): number;
|
|
82
|
+
export {};
|
|
83
|
+
//# sourceMappingURL=search.d.ts.map
|
package/lib/search.js
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
/** BM25 term-saturation constant (Okapi k1). */
|
|
2
|
+
export const BM25_K1 = 1.5;
|
|
3
|
+
/** BM25 document-length normalization constant (Okapi b). */
|
|
4
|
+
export const BM25_B = 0.75;
|
|
5
|
+
/** Multiplier for title-field matches (title hits outweigh body hits 2:1). */
|
|
6
|
+
export const TITLE_FIELD_WEIGHT = 2;
|
|
7
|
+
/**
|
|
8
|
+
* One pass, two run shapes: ASCII alphanumeric words, or maximal CJK runs.
|
|
9
|
+
* Runs never mix scripts because the alternation splits at the boundary, so
|
|
10
|
+
* 「深色主题dark主题」 tokenizes as 深色/色主/主题, "dark", 主题.
|
|
11
|
+
*/
|
|
12
|
+
const TOKEN_RUN_PATTERN = /[a-z0-9]+|[\u4e00-\u9fff]+/g;
|
|
13
|
+
const ASCII_RUN_PATTERN = /^[a-z0-9]+$/;
|
|
14
|
+
/**
|
|
15
|
+
* Tokenize text for relevance scoring: lowercase ASCII runs become word
|
|
16
|
+
* tokens; CJK runs become overlapping character bigrams (a single-character
|
|
17
|
+
* run stays a unigram). Deterministic and allocation-light — safe to call per
|
|
18
|
+
* injection build.
|
|
19
|
+
*
|
|
20
|
+
* @param text Arbitrary entry or query text.
|
|
21
|
+
* @returns Tokens in occurrence order (duplicates kept; consumers dedupe when
|
|
22
|
+
* order-independent weighting is wanted).
|
|
23
|
+
*/
|
|
24
|
+
export function tokenize(text) {
|
|
25
|
+
const tokens = [];
|
|
26
|
+
for (const match of text.toLowerCase().matchAll(TOKEN_RUN_PATTERN)) {
|
|
27
|
+
const run = match[0];
|
|
28
|
+
if (ASCII_RUN_PATTERN.test(run)) {
|
|
29
|
+
tokens.push(run);
|
|
30
|
+
}
|
|
31
|
+
else if (run.length === 1) {
|
|
32
|
+
tokens.push(run);
|
|
33
|
+
}
|
|
34
|
+
else {
|
|
35
|
+
for (let i = 0; i < run.length - 1; i += 1) {
|
|
36
|
+
tokens.push(run.slice(i, i + 2));
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return tokens;
|
|
41
|
+
}
|
|
42
|
+
function buildFieldStats(fieldTokens) {
|
|
43
|
+
const tfs = fieldTokens.map((tokens) => {
|
|
44
|
+
const tf = new Map();
|
|
45
|
+
for (const token of tokens) {
|
|
46
|
+
tf.set(token, (tf.get(token) ?? 0) + 1);
|
|
47
|
+
}
|
|
48
|
+
return tf;
|
|
49
|
+
});
|
|
50
|
+
const df = new Map();
|
|
51
|
+
let totalLength = 0;
|
|
52
|
+
for (let position = 0; position < tfs.length; position += 1) {
|
|
53
|
+
const tf = tfs[position];
|
|
54
|
+
let length = 0;
|
|
55
|
+
for (const count of tf.values()) {
|
|
56
|
+
length += count;
|
|
57
|
+
}
|
|
58
|
+
totalLength += length;
|
|
59
|
+
for (const token of tf.keys()) {
|
|
60
|
+
df.set(token, (df.get(token) ?? 0) + 1);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return {
|
|
64
|
+
tfs,
|
|
65
|
+
lengths: fieldTokens.map((tokens) => tokens.length),
|
|
66
|
+
df,
|
|
67
|
+
averageLength: tfs.length === 0 ? 0 : totalLength / tfs.length,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Build the corpus statistics (per-field tf, df, average length) for one
|
|
72
|
+
* ranking call. Pure: the index reads its input snapshot and never mutates
|
|
73
|
+
* the entries.
|
|
74
|
+
*
|
|
75
|
+
* @param entries The candidate entries being ranked this call.
|
|
76
|
+
* @returns An index that answers {@link relevanceScore} for exactly these
|
|
77
|
+
* entries (identity-keyed).
|
|
78
|
+
*/
|
|
79
|
+
export function buildRelevanceIndex(entries) {
|
|
80
|
+
const titles = [];
|
|
81
|
+
const bodies = [];
|
|
82
|
+
for (const entry of entries) {
|
|
83
|
+
titles.push(tokenize(entry.title));
|
|
84
|
+
bodies.push(tokenize(`${entry.content} ${entry.path}`));
|
|
85
|
+
}
|
|
86
|
+
return {
|
|
87
|
+
size: entries.length,
|
|
88
|
+
positions: new Map(entries.map((entry, position) => [entry, position])),
|
|
89
|
+
titles: buildFieldStats(titles),
|
|
90
|
+
bodies: buildFieldStats(bodies),
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
/** One field's BM25 contribution for a single term (0 when the term is absent). */
|
|
94
|
+
function bm25Term(termFrequency, documentFrequency, corpusSize, docLength, averageLength) {
|
|
95
|
+
if (termFrequency <= 0 || documentFrequency <= 0) {
|
|
96
|
+
return 0;
|
|
97
|
+
}
|
|
98
|
+
const inverseDocumentFrequency = Math.log(1 + (corpusSize - documentFrequency + 0.5) / (documentFrequency + 0.5));
|
|
99
|
+
const normalization = averageLength > 0 ? 1 - BM25_B + BM25_B * (docLength / averageLength) : 1;
|
|
100
|
+
return inverseDocumentFrequency * ((termFrequency * (BM25_K1 + 1)) / (termFrequency + BM25_K1 * normalization));
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Field-weighted BM25 score of `entry` against `query` within `index`:
|
|
104
|
+
* `TITLE_FIELD_WEIGHT × bm25(title) + bm25(body)`. Query tokens are deduped,
|
|
105
|
+
* so repeating a word adds no weight.
|
|
106
|
+
*
|
|
107
|
+
* @returns A score > 0 when at least one query token occurs in the entry
|
|
108
|
+
* (either field), and exactly 0 otherwise — the property rankEntries'
|
|
109
|
+
* relevance-first ordering relies on. Never throws; an entry outside
|
|
110
|
+
* the index simply scores 0.
|
|
111
|
+
*/
|
|
112
|
+
export function relevanceScore(index, entry, query) {
|
|
113
|
+
const position = index.positions.get(entry);
|
|
114
|
+
if (position === undefined || index.size === 0) {
|
|
115
|
+
return 0;
|
|
116
|
+
}
|
|
117
|
+
const seen = new Set();
|
|
118
|
+
let titleScore = 0;
|
|
119
|
+
let bodyScore = 0;
|
|
120
|
+
for (const token of tokenize(query)) {
|
|
121
|
+
if (seen.has(token)) {
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
seen.add(token);
|
|
125
|
+
const titleDf = index.titles.df.get(token) ?? 0;
|
|
126
|
+
if (titleDf > 0) {
|
|
127
|
+
titleScore += bm25Term(index.titles.tfs[position]?.get(token) ?? 0, titleDf, index.size, index.titles.lengths[position] ?? 0, index.titles.averageLength);
|
|
128
|
+
}
|
|
129
|
+
const bodyDf = index.bodies.df.get(token) ?? 0;
|
|
130
|
+
if (bodyDf > 0) {
|
|
131
|
+
bodyScore += bm25Term(index.bodies.tfs[position]?.get(token) ?? 0, bodyDf, index.size, index.bodies.lengths[position] ?? 0, index.bodies.averageLength);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
return TITLE_FIELD_WEIGHT * titleScore + bodyScore;
|
|
135
|
+
}
|
|
136
|
+
//# sourceMappingURL=search.js.map
|
package/lib/service.js
CHANGED
|
@@ -1,15 +1,73 @@
|
|
|
1
|
+
import { CONFLICT_HINT_KEY } from "./types.js";
|
|
1
2
|
import { applyRefinementProposal } from "./apply.js";
|
|
2
3
|
import { randomUUID } from "node:crypto";
|
|
3
4
|
import { rollbackProposal } from "./rollback.js";
|
|
4
5
|
import { loadHarnessState, saveHarnessState } from "./state.js";
|
|
5
6
|
import { appendResult, loadResults, snapshotBefore, storePaths } from "./store.js";
|
|
7
|
+
import { CONFLICT_BLOCK_SCORE, CONFLICT_WARN_SCORE, buildConflictNotice, mostSimilarEntry, secretLeakReason } from "./promotion.js";
|
|
6
8
|
export function createEvolutionEngine(baseDir, hooks = {}) {
|
|
7
9
|
function load(scope, sessionId) {
|
|
8
10
|
return loadHarnessState(storePaths(baseDir, scope, sessionId).stateDir, scope);
|
|
9
11
|
}
|
|
10
12
|
function apply(scope, sessionId, proposal, context) {
|
|
11
13
|
const paths = storePaths(baseDir, scope, sessionId);
|
|
12
|
-
|
|
14
|
+
// Working state is ALWAYS the freshest on-disk snapshot: apply mutates
|
|
15
|
+
// and persists the state whole-file, so building on the caller's
|
|
16
|
+
// planning-time copy would silently overwrite concurrent writers
|
|
17
|
+
// (another gate run, another session writing global). The caller's
|
|
18
|
+
// baselineState is ONLY the optimistic-concurrency comparison baseline
|
|
19
|
+
// — "reject edits whose target changed since planning" (review audit
|
|
20
|
+
// 2026-08-28 B1: the two roles were previously folded into one object,
|
|
21
|
+
// which made the advertised guard unreachable).
|
|
22
|
+
const state = load(scope, sessionId);
|
|
23
|
+
// Write-time conflict guard (R2): global creates are checked against
|
|
24
|
+
// the existing same-kind entries BEFORE any side effect — a
|
|
25
|
+
// near-duplicate is rejected with an actionable error (evolve_update
|
|
26
|
+
// instead), a moderate overlap proceeds stamped with
|
|
27
|
+
// CONFLICT_HINT_KEY. Rollbacks bypass the guard: re-creating an entry
|
|
28
|
+
// that resembles its successor is the point of rollback. Local scope
|
|
29
|
+
// is never blocked (scratch space); the wrapup/fate promotion path
|
|
30
|
+
// already enforces its own overlap policy there.
|
|
31
|
+
//
|
|
32
|
+
// Secret-leak guard (P0, same throat): global creates AND updates are
|
|
33
|
+
// screened for credential-shaped literals before any side effect — a
|
|
34
|
+
// secret reaching the cross-session store is a leak even when the
|
|
35
|
+
// entry itself is legitimate. The screen covers every field the edit
|
|
36
|
+
// can plant: title, content, and the JSON forms of reference,
|
|
37
|
+
// arguments, and metadata (mount embeds reference verbatim into the
|
|
38
|
+
// generated plugin file). Fixed patterns, not policy-configurable.
|
|
39
|
+
const warnHits = new Map();
|
|
40
|
+
if (scope === "global" && !context?.rollbackOf) {
|
|
41
|
+
for (const [index, edit] of proposal.edits.entries()) {
|
|
42
|
+
if (edit.action === "create" || edit.action === "update") {
|
|
43
|
+
const screenable = [
|
|
44
|
+
typeof edit.title === "string" ? edit.title : "",
|
|
45
|
+
typeof edit.content === "string" ? edit.content : "",
|
|
46
|
+
edit.reference !== undefined ? JSON.stringify(edit.reference) : "",
|
|
47
|
+
edit.arguments !== undefined ? JSON.stringify(edit.arguments) : "",
|
|
48
|
+
edit.metadata !== undefined ? JSON.stringify(edit.metadata) : "",
|
|
49
|
+
];
|
|
50
|
+
const secret = secretLeakReason(screenable.join("\n"));
|
|
51
|
+
if (secret) {
|
|
52
|
+
throw new Error(`${edit.action} blocked: ${secret}`);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
if (edit.action !== "create")
|
|
56
|
+
continue;
|
|
57
|
+
// An unknown kind must fail per-edit in validateEdit, never
|
|
58
|
+
// crash the whole proposal here (review audit 2026-08-28 S1).
|
|
59
|
+
const corpus = state.entries[edit.kind];
|
|
60
|
+
if (!corpus)
|
|
61
|
+
continue;
|
|
62
|
+
const hit = mostSimilarEntry(Object.values(corpus), edit.title ?? "", edit.content ?? "", CONFLICT_WARN_SCORE);
|
|
63
|
+
if (!hit)
|
|
64
|
+
continue;
|
|
65
|
+
if (hit.score >= CONFLICT_BLOCK_SCORE) {
|
|
66
|
+
throw new Error(`create blocked: ${buildConflictNotice(hit)} already lives in the global ${edit.kind} store — use evolve_update on it instead of adding a duplicate`);
|
|
67
|
+
}
|
|
68
|
+
warnHits.set(index, hit);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
13
71
|
const id = `evolve_${Date.now().toString(36)}_${randomUUID().slice(0, 8)}`;
|
|
14
72
|
// Code-enforced snapshot: runs before any mutation, cannot be skipped by the model.
|
|
15
73
|
snapshotBefore(paths, id);
|
|
@@ -20,6 +78,21 @@ export function createEvolutionEngine(baseDir, hooks = {}) {
|
|
|
20
78
|
...(context?.baselineState ? { baselineState: context.baselineState } : {}),
|
|
21
79
|
...(context?.rollbackOf ? { rollbackOf: context.rollbackOf } : {}),
|
|
22
80
|
});
|
|
81
|
+
// Stamp warn-tier conflicts onto the freshly created entries (both the
|
|
82
|
+
// live state and the result's after-snapshot stay coherent).
|
|
83
|
+
for (const [index, hit] of warnHits) {
|
|
84
|
+
const applied = result.appliedEdits[index];
|
|
85
|
+
if (!applied?.applied || applied.action !== "create" || !applied.id)
|
|
86
|
+
continue;
|
|
87
|
+
const hint = `${applied.kind}:${hit.id}:${hit.score.toFixed(2)}`;
|
|
88
|
+
const live = state.entries[applied.kind][applied.id];
|
|
89
|
+
if (live) {
|
|
90
|
+
live.metadata[CONFLICT_HINT_KEY] = hint;
|
|
91
|
+
if (applied.after) {
|
|
92
|
+
applied.after.metadata[CONFLICT_HINT_KEY] = hint;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
23
96
|
saveHarnessState(paths.stateDir, state);
|
|
24
97
|
appendResult(paths, result);
|
|
25
98
|
hooks.onApplied?.(result);
|
package/lib/state.d.ts
CHANGED
|
@@ -1,8 +1,4 @@
|
|
|
1
1
|
import type { HarnessScope, HarnessState } from "./types.js";
|
|
2
|
-
/** Directory holding the cross-session (global) store. */
|
|
3
|
-
export declare function globalStateDir(baseDir: string): string;
|
|
4
|
-
/** Directory holding a session-scoped (local) store, if the session has one. */
|
|
5
|
-
export declare function localStateDir(sessionDir: string | undefined): string | undefined;
|
|
6
2
|
export declare function stateFilePath(stateDir: string): string;
|
|
7
3
|
/**
|
|
8
4
|
* Load state from disk, degrading to empty on any unreadable or malformed
|
|
@@ -21,14 +17,6 @@ export declare function mergeHarnessStates(globalState: HarnessState, localState
|
|
|
21
17
|
* preserving the mode of an existing file (defaults to 0o600 for new files).
|
|
22
18
|
*/
|
|
23
19
|
export declare function saveHarnessState(stateDir: string, state: HarnessState): string;
|
|
24
|
-
/**
|
|
25
|
-
* Capture the state snapshot a plan was based on. At apply time the caller
|
|
26
|
-
* re-reads the file and compares; an entry that changed since planning is
|
|
27
|
-
* rejected per-edit, never silently overwritten.
|
|
28
|
-
*/
|
|
29
|
-
export declare function baselineOf(state: HarnessState): HarnessState;
|
|
30
20
|
/** True when an entry in `current` differs from the same entry in `baseline`. */
|
|
31
21
|
export declare function entryChangedSince(baseline: HarnessState, current: HarnessState, kind: keyof HarnessState["entries"], id: string): boolean;
|
|
32
|
-
/** The set of keys an entry must not expose beyond the persisted shape. */
|
|
33
|
-
export declare const ENTRY_KEYS: readonly ["id", "kind", "title", "content", "path", "scope", "reference", "arguments", "metadata", "source", "created_at", "updated_at", "version"];
|
|
34
22
|
//# sourceMappingURL=state.d.ts.map
|
package/lib/state.js
CHANGED
|
@@ -16,14 +16,6 @@ import { readFileSync } from "node:fs";
|
|
|
16
16
|
import { randomUUID } from "node:crypto";
|
|
17
17
|
import { join } from "node:path";
|
|
18
18
|
import { emptyHarnessState } from "./types.js";
|
|
19
|
-
/** Directory holding the cross-session (global) store. */
|
|
20
|
-
export function globalStateDir(baseDir) {
|
|
21
|
-
return join(baseDir, "evolve");
|
|
22
|
-
}
|
|
23
|
-
/** Directory holding a session-scoped (local) store, if the session has one. */
|
|
24
|
-
export function localStateDir(sessionDir) {
|
|
25
|
-
return sessionDir ? join(sessionDir, "evolve") : undefined;
|
|
26
|
-
}
|
|
27
19
|
export function stateFilePath(stateDir) {
|
|
28
20
|
return join(stateDir, "harness_state.json");
|
|
29
21
|
}
|
|
@@ -95,7 +87,10 @@ export function loadHarnessState(stateDir, scope = "global") {
|
|
|
95
87
|
}
|
|
96
88
|
}
|
|
97
89
|
if (Array.isArray(root["refinements"])) {
|
|
98
|
-
|
|
90
|
+
// Shape-check members like entries do: a hand-edited file must never
|
|
91
|
+
// reach render/planner paths and crash them (review audit 2026-08-28
|
|
92
|
+
// S6 — loadHarnessState promises "never throws on a bad file").
|
|
93
|
+
state.refinements = root["refinements"].filter((item) => typeof item === "object" && item !== null && typeof item["id"] === "string");
|
|
99
94
|
}
|
|
100
95
|
return state;
|
|
101
96
|
}
|
|
@@ -140,20 +135,10 @@ export function saveHarnessState(stateDir, state) {
|
|
|
140
135
|
}
|
|
141
136
|
return path;
|
|
142
137
|
}
|
|
143
|
-
/**
|
|
144
|
-
* Capture the state snapshot a plan was based on. At apply time the caller
|
|
145
|
-
* re-reads the file and compares; an entry that changed since planning is
|
|
146
|
-
* rejected per-edit, never silently overwritten.
|
|
147
|
-
*/
|
|
148
|
-
export function baselineOf(state) {
|
|
149
|
-
return JSON.parse(JSON.stringify(state));
|
|
150
|
-
}
|
|
151
138
|
/** True when an entry in `current` differs from the same entry in `baseline`. */
|
|
152
139
|
export function entryChangedSince(baseline, current, kind, id) {
|
|
153
140
|
const before = baseline.entries[kind][id];
|
|
154
141
|
const after = current.entries[kind][id];
|
|
155
142
|
return JSON.stringify(before ?? null) !== JSON.stringify(after ?? null);
|
|
156
143
|
}
|
|
157
|
-
/** The set of keys an entry must not expose beyond the persisted shape. */
|
|
158
|
-
export const ENTRY_KEYS = ["id", "kind", "title", "content", "path", "scope", "reference", "arguments", "metadata", "source", "created_at", "updated_at", "version"];
|
|
159
144
|
//# sourceMappingURL=state.js.map
|
package/lib/store.js
CHANGED
|
@@ -32,7 +32,9 @@ export function snapshotBefore(paths, refinementId) {
|
|
|
32
32
|
return;
|
|
33
33
|
}
|
|
34
34
|
mkdirSync(paths.snapshotsDir, { recursive: true });
|
|
35
|
-
|
|
35
|
+
// 0600: the snapshot is a full copy of harness_state.json — same
|
|
36
|
+
// permission discipline as the store itself (review audit 2026-08-28 S5).
|
|
37
|
+
writeFileSync(join(paths.snapshotsDir, `${refinementId}.json`), readFileSync(statePath, "utf8"), { encoding: "utf8", mode: 0o600 });
|
|
36
38
|
}
|
|
37
39
|
/** Append an applied result to the store's JSONL history. */
|
|
38
40
|
export function appendResult(paths, result) {
|