auxilo-mcp 0.9.3 → 0.9.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -5
- package/config/near-duplicate.json +9 -0
- package/lib/extraction-index.js +462 -0
- package/lib/installer.js +3 -0
- package/lib/sensitivity-filter.js +48 -1
- package/lib/similarity.js +275 -0
- package/mcp-server.js +66 -22
- package/package.json +6 -2
- package/scripts/extract-local.js +472 -29
- package/scripts/runner.js +80 -8
- package/scripts/sources/source.interface.js +84 -1
package/README.md
CHANGED
|
@@ -4,9 +4,9 @@
|
|
|
4
4
|
[](https://www.npmjs.com/package/auxilo-mcp)
|
|
5
5
|
[](LICENSE)
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
Your agent already solved this. It found the fix, shipped, and lost it when the session ended. Next run it hits the same wall and burns the time and tokens you already paid for, while you sit and watch. Auxilo stops that. Your agent stops solving the same problem twice.
|
|
8
8
|
|
|
9
|
-
Your agent
|
|
9
|
+
Auxilo is an MCP server that auto-extracts operational learnings from your coding agent's sessions, gives your agent its own learnings back in every later session, and lists them in a marketplace where other agents pay to unlock them. Your agent's own learnings always come back at $0. When another agent unlocks what yours figured out, you earn.
|
|
10
10
|
|
|
11
11
|
## The problem
|
|
12
12
|
|
|
@@ -92,7 +92,7 @@ The same block works anywhere MCP configs are read. The installer also detects C
|
|
|
92
92
|
|
|
93
93
|
## Tools
|
|
94
94
|
|
|
95
|
-
|
|
95
|
+
17 tools:
|
|
96
96
|
|
|
97
97
|
| Tool | What it does | Cost |
|
|
98
98
|
|---|---|---|
|
|
@@ -105,7 +105,6 @@ The same block works anywhere MCP configs are read. The installer also detects C
|
|
|
105
105
|
| `auxilo_skill` | Connection details, auth, and pricing for one skill | Free |
|
|
106
106
|
| `auxilo_categories` | List categories with counts | Free |
|
|
107
107
|
| `auxilo_stats` | Registry statistics | Free |
|
|
108
|
-
| `get_stats` | Registry statistics, alias | Free |
|
|
109
108
|
| `get_knowledge_stats` | Marketplace statistics | Free |
|
|
110
109
|
| `auxilo_contributor` | Earnings for a contributor wallet | Free |
|
|
111
110
|
| `auxilo_account_earnings` | Earnings and pending balance for your account | Free |
|
|
@@ -153,7 +152,7 @@ Unlocks (`GET /knowledge/:id`, minimum $0.05) are paid with [x402](https://www.x
|
|
|
153
152
|
|
|
154
153
|
- OpenAPI spec: [auxilo.io/openapi.json](https://auxilo.io/openapi.json)
|
|
155
154
|
- Agent discovery card: `https://auxilo.io/.well-known/agent.json`
|
|
156
|
-
- Categories: data-processing, web-interaction, code-execution,
|
|
155
|
+
- Categories: data-processing, web-interaction, code-execution, storage-state, payment-financial, monitoring. Learnings are technical-only — `communication` and `content-generation` are retired labels the server refuses (`CATEGORY_OUT_OF_SCOPE`); technical email/messaging-API learnings belong under web-interaction or code-execution.
|
|
157
156
|
|
|
158
157
|
## Privacy
|
|
159
158
|
|
|
@@ -0,0 +1,462 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Client-side extraction memory for SPEC3-F2.
|
|
5
|
+
*
|
|
6
|
+
* The index is local, append-only, and advisory. Every read or write failure
|
|
7
|
+
* fails open: extraction/submission continues without prompt memory or lexical
|
|
8
|
+
* filtering. The only shared detector is lib/similarity.js.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
const crypto = require('node:crypto');
|
|
12
|
+
const fs = require('node:fs');
|
|
13
|
+
const os = require('node:os');
|
|
14
|
+
const path = require('node:path');
|
|
15
|
+
const { findNearDuplicate, scoreChannels } = require('./similarity.js');
|
|
16
|
+
|
|
17
|
+
const DEFAULT_INDEX_PATH = path.join(os.homedir(), '.auxilo', 'extracted-index.jsonl');
|
|
18
|
+
const HYDRATION_PAGE_SIZE = 500;
|
|
19
|
+
const PROMPT_MEMORY_MAX_TOKENS = 1200;
|
|
20
|
+
const PROMPT_MEMORY_MAX_ROWS = 40;
|
|
21
|
+
const VALID_STATUSES = new Set(['approved', 'rejected', 'pending_review']);
|
|
22
|
+
|
|
23
|
+
const MEMORY_INSTRUCTIONS = `PREVIOUSLY CAPTURED LESSONS
|
|
24
|
+
The following lessons were already submitted by this account.
|
|
25
|
+
A candidate that re-states any listed lesson — reworded, or a different facet of the same operational insight — is DROPPED, not relabeled and not "improved."
|
|
26
|
+
Extract new facts about a listed lesson ONLY when the new fact would change what another agent does: it must create a behavioral difference, not merely a wording difference.
|
|
27
|
+
Never discard a memory match invisibly. Put the complete candidate plus the matched lesson id/title in dedup_drops so the local runner can write its audit trail.`;
|
|
28
|
+
|
|
29
|
+
const CATEGORY_HINTS = Object.freeze({
|
|
30
|
+
'data-processing': ['data', 'parse', 'parser', 'json', 'csv', 'transform', 'pipeline'],
|
|
31
|
+
'web-interaction': ['api', 'http', 'browser', 'gmail', 'webhook', 'request', 'response'],
|
|
32
|
+
'code-execution': ['code', 'script', 'python', 'node', 'shell', 'command', 'runtime'],
|
|
33
|
+
'storage-state': ['state', 'queue', 'database', 'storage', 'file', 'commit', 'cache'],
|
|
34
|
+
'payment-financial': ['payment', 'stripe', 'usdc', 'wallet', 'settlement', 'withdraw'],
|
|
35
|
+
monitoring: ['monitor', 'alert', 'health', 'log', 'metric', 'observability', 'incident'],
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
function loud(log, message) {
|
|
39
|
+
try {
|
|
40
|
+
(log || console.error)(`[extraction-index] ${message}`);
|
|
41
|
+
} catch {
|
|
42
|
+
// A broken logger must not turn advisory dedup into a submission blocker.
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function normalizeTags(tags) {
|
|
47
|
+
return Array.isArray(tags) ? tags.slice(0, 8).map(String) : [];
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function validIndexRow(row) {
|
|
51
|
+
return Boolean(
|
|
52
|
+
row &&
|
|
53
|
+
typeof row === 'object' &&
|
|
54
|
+
typeof row.title === 'string' &&
|
|
55
|
+
row.title.trim() &&
|
|
56
|
+
typeof row.category === 'string'
|
|
57
|
+
);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Read an append-only JSONL index. A malformed line is skipped and logged, but
|
|
62
|
+
* marks the read unusable so extraction fails open instead of trusting a
|
|
63
|
+
* partially corrupt memory/filter corpus.
|
|
64
|
+
*/
|
|
65
|
+
function readExtractionIndex(opts = {}) {
|
|
66
|
+
const indexPath = opts.indexPath || DEFAULT_INDEX_PATH;
|
|
67
|
+
const fsImpl = opts.fsImpl || fs;
|
|
68
|
+
let exists;
|
|
69
|
+
try {
|
|
70
|
+
exists = fsImpl.existsSync(indexPath);
|
|
71
|
+
} catch (error) {
|
|
72
|
+
loud(opts.log, `index existence check failed; dedup disabled for this run: ${error.message}`);
|
|
73
|
+
return { state: 'unreadable', usable: false, rows: [], bad_lines: 0 };
|
|
74
|
+
}
|
|
75
|
+
if (!exists) return { state: 'missing', usable: false, rows: [], bad_lines: 0 };
|
|
76
|
+
|
|
77
|
+
let raw;
|
|
78
|
+
try {
|
|
79
|
+
raw = fsImpl.readFileSync(indexPath, 'utf8');
|
|
80
|
+
} catch (error) {
|
|
81
|
+
loud(opts.log, `index read failed; dedup disabled for this run: ${error.message}`);
|
|
82
|
+
return { state: 'unreadable', usable: false, rows: [], bad_lines: 0 };
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const rows = [];
|
|
86
|
+
let badLines = 0;
|
|
87
|
+
const lines = String(raw).split(/\r?\n/);
|
|
88
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
89
|
+
const line = lines[index].trim();
|
|
90
|
+
if (!line) continue;
|
|
91
|
+
try {
|
|
92
|
+
const row = JSON.parse(line);
|
|
93
|
+
if (!validIndexRow(row)) throw new Error('missing title/category');
|
|
94
|
+
rows.push(row);
|
|
95
|
+
} catch (error) {
|
|
96
|
+
badLines += 1;
|
|
97
|
+
loud(opts.log, `skipping corrupt line ${index + 1} in ${indexPath}: ${error.message}`);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
if (badLines > 0) {
|
|
102
|
+
loud(opts.log, `index contains ${badLines} corrupt line(s); prompt memory and lexical dedup are disabled for this run`);
|
|
103
|
+
return { state: 'corrupt', usable: false, rows, bad_lines: badLines };
|
|
104
|
+
}
|
|
105
|
+
return { state: 'ready', usable: true, rows, bad_lines: 0 };
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function appendJsonlRows(rows, opts = {}) {
|
|
109
|
+
const indexPath = opts.indexPath || DEFAULT_INDEX_PATH;
|
|
110
|
+
const fsImpl = opts.fsImpl || fs;
|
|
111
|
+
try {
|
|
112
|
+
fsImpl.mkdirSync(path.dirname(indexPath), { recursive: true, mode: 0o700 });
|
|
113
|
+
if (!rows.length) {
|
|
114
|
+
const fd = fsImpl.openSync(indexPath, 'a', 0o600);
|
|
115
|
+
fsImpl.closeSync(fd);
|
|
116
|
+
return true;
|
|
117
|
+
}
|
|
118
|
+
const payload = rows.map((row) => JSON.stringify(row)).join('\n') + '\n';
|
|
119
|
+
fsImpl.appendFileSync(indexPath, payload, { encoding: 'utf8', mode: 0o600 });
|
|
120
|
+
return true;
|
|
121
|
+
} catch (error) {
|
|
122
|
+
loud(opts.log, `index append failed; extraction/submission continues: ${error.message}`);
|
|
123
|
+
return false;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function localIndexRow(learning, response = {}, opts = {}) {
|
|
128
|
+
const nowValue = typeof opts.now === 'function'
|
|
129
|
+
? opts.now()
|
|
130
|
+
: (opts.now || new Date().toISOString());
|
|
131
|
+
const body = String(learning.body || '');
|
|
132
|
+
return {
|
|
133
|
+
title: String(learning.title || ''),
|
|
134
|
+
category: String(learning.category || ''),
|
|
135
|
+
tags: normalizeTags(learning.tags),
|
|
136
|
+
body_hash: crypto.createHash('sha256').update(body).digest('hex'),
|
|
137
|
+
body,
|
|
138
|
+
submitted_at: String(nowValue),
|
|
139
|
+
...(typeof response.id === 'string' && response.id && { learning_id: response.id }),
|
|
140
|
+
...(typeof response.status === 'string' && VALID_STATUSES.has(response.status) && {
|
|
141
|
+
status: response.status,
|
|
142
|
+
}),
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Append only after POST /learn returns 2xx. A network/4xx failure remains
|
|
148
|
+
* retryable and is not allowed to poison the local dedup memory.
|
|
149
|
+
*/
|
|
150
|
+
function appendSubmittedLearning(learning, response = {}, opts = {}) {
|
|
151
|
+
if (!learning || typeof learning !== 'object') return false;
|
|
152
|
+
return appendJsonlRows([localIndexRow(learning, response, opts)], opts);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function hydratedIndexRow(learning) {
|
|
156
|
+
return {
|
|
157
|
+
title: String(learning.title || ''),
|
|
158
|
+
category: String(learning.category || ''),
|
|
159
|
+
tags: normalizeTags(learning.tags),
|
|
160
|
+
body_hash: null,
|
|
161
|
+
submitted_at: learning.created_at || null,
|
|
162
|
+
...(typeof learning.id === 'string' && learning.id && { learning_id: learning.id }),
|
|
163
|
+
...(typeof learning.status === 'string' && VALID_STATUSES.has(learning.status) && {
|
|
164
|
+
status: learning.status,
|
|
165
|
+
}),
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Hydrate only a missing index from the caller's own metadata endpoint. The
|
|
171
|
+
* endpoint intentionally carries no body, so hydrated rows inform the prompt
|
|
172
|
+
* only; lexical filtering is limited to locally appended rows with a body.
|
|
173
|
+
*/
|
|
174
|
+
async function hydrateExtractionIndex(opts = {}) {
|
|
175
|
+
const indexPath = opts.indexPath || DEFAULT_INDEX_PATH;
|
|
176
|
+
const fsImpl = opts.fsImpl || fs;
|
|
177
|
+
try {
|
|
178
|
+
if (fsImpl.existsSync(indexPath)) return { hydrated: false, reason: 'not_fresh' };
|
|
179
|
+
} catch (error) {
|
|
180
|
+
loud(opts.log, `fresh-index check failed; hydration skipped: ${error.message}`);
|
|
181
|
+
return { hydrated: false, reason: 'unreadable' };
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
const fetchImpl = opts.fetchImpl || globalThis.fetch;
|
|
185
|
+
const baseUrl = String(opts.baseUrl || '').replace(/\/+$/, '');
|
|
186
|
+
const apiKey = opts.apiKey;
|
|
187
|
+
if (typeof fetchImpl !== 'function' || !baseUrl || !apiKey) {
|
|
188
|
+
loud(opts.log, 'fresh-machine hydration unavailable (missing fetch, base URL, or API key); extracting without memory');
|
|
189
|
+
return { hydrated: false, reason: 'unavailable' };
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
const hydratedRows = [];
|
|
193
|
+
let offset = 0;
|
|
194
|
+
try {
|
|
195
|
+
for (let page = 0; page < 10000; page += 1) {
|
|
196
|
+
const response = await fetchImpl(
|
|
197
|
+
`${baseUrl}/account/learnings?limit=${HYDRATION_PAGE_SIZE}&offset=${offset}`,
|
|
198
|
+
{ headers: { 'X-API-Key': apiKey } }
|
|
199
|
+
);
|
|
200
|
+
if (!response || !response.ok) {
|
|
201
|
+
const status = response && Number.isFinite(response.status) ? response.status : 'unknown';
|
|
202
|
+
throw new Error(`GET /account/learnings returned ${status}`);
|
|
203
|
+
}
|
|
204
|
+
const payload = await response.json();
|
|
205
|
+
if (!payload || !Array.isArray(payload.learnings)) {
|
|
206
|
+
throw new Error('GET /account/learnings returned an invalid payload');
|
|
207
|
+
}
|
|
208
|
+
for (const row of payload.learnings) {
|
|
209
|
+
const hydrated = hydratedIndexRow(row);
|
|
210
|
+
if (!validIndexRow(hydrated)) {
|
|
211
|
+
loud(opts.log, 'skipping malformed hydration row from GET /account/learnings');
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
214
|
+
hydratedRows.push(hydrated);
|
|
215
|
+
}
|
|
216
|
+
offset += payload.learnings.length;
|
|
217
|
+
const total = Number.isFinite(payload.total) ? payload.total : null;
|
|
218
|
+
if (payload.learnings.length < HYDRATION_PAGE_SIZE ||
|
|
219
|
+
(total !== null && offset >= total)) break;
|
|
220
|
+
if (payload.learnings.length === 0) break;
|
|
221
|
+
}
|
|
222
|
+
} catch (error) {
|
|
223
|
+
loud(opts.log, `fresh-machine hydration failed; extracting without memory: ${error.message}`);
|
|
224
|
+
return { hydrated: false, reason: 'request_failed' };
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
if (!appendJsonlRows(hydratedRows, { indexPath, fsImpl, log: opts.log })) {
|
|
228
|
+
return { hydrated: false, reason: 'write_failed' };
|
|
229
|
+
}
|
|
230
|
+
loud(opts.log, `hydrated ${hydratedRows.length} own learning metadata row(s) into ${indexPath}`);
|
|
231
|
+
return { hydrated: true, count: hydratedRows.length };
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
async function loadIndexForExtraction(opts = {}) {
|
|
235
|
+
const first = readExtractionIndex(opts);
|
|
236
|
+
if (first.state !== 'missing') return first;
|
|
237
|
+
const hydration = await hydrateExtractionIndex(opts);
|
|
238
|
+
if (!hydration.hydrated) return first;
|
|
239
|
+
return readExtractionIndex(opts);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function estimatePromptTokens(text) {
|
|
243
|
+
return Math.ceil(Buffer.byteLength(String(text || ''), 'utf8') / 4);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
function inferCategoryHints(transcript) {
|
|
247
|
+
const text = String(transcript || '').toLowerCase();
|
|
248
|
+
const scores = [];
|
|
249
|
+
for (const [category, words] of Object.entries(CATEGORY_HINTS)) {
|
|
250
|
+
let score = 0;
|
|
251
|
+
for (const word of words) {
|
|
252
|
+
const matches = text.match(new RegExp(`\\b${word.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'g'));
|
|
253
|
+
score += matches ? matches.length : 0;
|
|
254
|
+
}
|
|
255
|
+
if (score > 0) scores.push({ category, score });
|
|
256
|
+
}
|
|
257
|
+
scores.sort((a, b) => b.score - a.score || a.category.localeCompare(b.category));
|
|
258
|
+
if (!scores.length) return [];
|
|
259
|
+
const top = scores[0].score;
|
|
260
|
+
return scores.filter((entry) => entry.score === top).map((entry) => entry.category);
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
function submittedTime(row) {
|
|
264
|
+
const parsed = Date.parse(row.submitted_at || row.created_at || '');
|
|
265
|
+
return Number.isFinite(parsed) ? parsed : 0;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
function oneLineGist(row) {
|
|
269
|
+
if (typeof row.body === 'string' && row.body.trim()) {
|
|
270
|
+
const compact = row.body.replace(/\s+/g, ' ').trim();
|
|
271
|
+
const sentence = compact.match(/^(.{1,220}?[.!?])(?:\s|$)/);
|
|
272
|
+
return (sentence ? sentence[1] : compact.slice(0, 220)).trim();
|
|
273
|
+
}
|
|
274
|
+
const tags = normalizeTags(row.tags).filter(Boolean);
|
|
275
|
+
if (tags.length) return `Previously captured ${row.category} lesson tagged ${tags.join(', ')}.`;
|
|
276
|
+
return `Previously captured ${row.category} lesson.`;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
function indexRowId(row, index = 0) {
|
|
280
|
+
return String(
|
|
281
|
+
row.learning_id ||
|
|
282
|
+
row.id ||
|
|
283
|
+
row.body_hash ||
|
|
284
|
+
`local-index-${index}`
|
|
285
|
+
);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
function promptMemoryLine(row, index) {
|
|
289
|
+
const title = String(row.title || '').replace(/\s+/g, ' ').trim().slice(0, 180);
|
|
290
|
+
const gist = oneLineGist(row).slice(0, 240);
|
|
291
|
+
return `- [id:${indexRowId(row, index)} category:${row.category}] ${title} — ${gist}`;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/**
|
|
295
|
+
* Budget strategy: inferred same-category rows first, then most recent rows,
|
|
296
|
+
* with deterministic title/ID tie-breaking. The instruction header and rows
|
|
297
|
+
* together may never exceed maxTokens.
|
|
298
|
+
*/
|
|
299
|
+
function buildPromptMemory(rows, opts = {}) {
|
|
300
|
+
const maxTokens = Number.isFinite(opts.maxTokens)
|
|
301
|
+
? Math.max(0, Math.floor(opts.maxTokens))
|
|
302
|
+
: PROMPT_MEMORY_MAX_TOKENS;
|
|
303
|
+
const maxRows = Number.isFinite(opts.maxRows)
|
|
304
|
+
? Math.max(0, Math.floor(opts.maxRows))
|
|
305
|
+
: PROMPT_MEMORY_MAX_ROWS;
|
|
306
|
+
const hints = new Set(opts.categoryHints || inferCategoryHints(opts.transcript));
|
|
307
|
+
const eligible = (Array.isArray(rows) ? rows : [])
|
|
308
|
+
.filter(validIndexRow)
|
|
309
|
+
.slice()
|
|
310
|
+
.sort((a, b) => {
|
|
311
|
+
const aMatch = hints.has(a.category) ? 1 : 0;
|
|
312
|
+
const bMatch = hints.has(b.category) ? 1 : 0;
|
|
313
|
+
return bMatch - aMatch ||
|
|
314
|
+
submittedTime(b) - submittedTime(a) ||
|
|
315
|
+
String(a.title).localeCompare(String(b.title)) ||
|
|
316
|
+
String(a.learning_id || '').localeCompare(String(b.learning_id || ''));
|
|
317
|
+
});
|
|
318
|
+
|
|
319
|
+
if (!eligible.length || maxRows === 0 || estimatePromptTokens(MEMORY_INSTRUCTIONS) > maxTokens) {
|
|
320
|
+
return {
|
|
321
|
+
section: '',
|
|
322
|
+
estimated_tokens: 0,
|
|
323
|
+
included_count: 0,
|
|
324
|
+
included_rows: [],
|
|
325
|
+
category_hints: [...hints],
|
|
326
|
+
};
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
const lines = [];
|
|
330
|
+
const includedRows = [];
|
|
331
|
+
for (let index = 0; index < eligible.length; index += 1) {
|
|
332
|
+
const row = eligible[index];
|
|
333
|
+
if (lines.length >= maxRows) break;
|
|
334
|
+
const candidateLines = [...lines, promptMemoryLine(row, index)];
|
|
335
|
+
const candidate = `${MEMORY_INSTRUCTIONS}\n${candidateLines.join('\n')}\n\n`;
|
|
336
|
+
if (estimatePromptTokens(candidate) > maxTokens) break;
|
|
337
|
+
lines.push(candidateLines[candidateLines.length - 1]);
|
|
338
|
+
includedRows.push({
|
|
339
|
+
id: indexRowId(row, index),
|
|
340
|
+
title: row.title,
|
|
341
|
+
category: row.category,
|
|
342
|
+
});
|
|
343
|
+
}
|
|
344
|
+
if (!lines.length) {
|
|
345
|
+
return {
|
|
346
|
+
section: '',
|
|
347
|
+
estimated_tokens: 0,
|
|
348
|
+
included_count: 0,
|
|
349
|
+
included_rows: [],
|
|
350
|
+
category_hints: [...hints],
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
const section = `${MEMORY_INSTRUCTIONS}\n${lines.join('\n')}\n\n`;
|
|
354
|
+
return {
|
|
355
|
+
section,
|
|
356
|
+
estimated_tokens: estimatePromptTokens(section),
|
|
357
|
+
included_count: lines.length,
|
|
358
|
+
included_rows: includedRows,
|
|
359
|
+
category_hints: [...hints],
|
|
360
|
+
};
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/**
|
|
364
|
+
* Rank index rows against one extracted candidate with the shared F1 scoring
|
|
365
|
+
* implementation. No flag threshold is applied: the score is used only for a
|
|
366
|
+
* deterministic candidate-anchored top-K list for the local judge.
|
|
367
|
+
*/
|
|
368
|
+
function rankIndexRowsForCandidate(candidate, rows, opts = {}) {
|
|
369
|
+
const topK = Number.isFinite(opts.topK)
|
|
370
|
+
? Math.max(0, Math.floor(opts.topK))
|
|
371
|
+
: 10;
|
|
372
|
+
if (!candidate || topK === 0) return [];
|
|
373
|
+
return (Array.isArray(rows) ? rows : [])
|
|
374
|
+
.filter(validIndexRow)
|
|
375
|
+
.map((row, index) => {
|
|
376
|
+
const ranked = {
|
|
377
|
+
id: indexRowId(row, index),
|
|
378
|
+
title: row.title,
|
|
379
|
+
category: row.category,
|
|
380
|
+
body: typeof row.body === 'string' ? row.body : '',
|
|
381
|
+
};
|
|
382
|
+
const channels = scoreChannels(candidate, ranked);
|
|
383
|
+
return {
|
|
384
|
+
id: ranked.id,
|
|
385
|
+
title: ranked.title,
|
|
386
|
+
category: ranked.category,
|
|
387
|
+
similarity: channels.composite,
|
|
388
|
+
channels,
|
|
389
|
+
};
|
|
390
|
+
})
|
|
391
|
+
.sort((a, b) =>
|
|
392
|
+
b.similarity - a.similarity ||
|
|
393
|
+
a.id.localeCompare(b.id) ||
|
|
394
|
+
String(a.title).localeCompare(String(b.title))
|
|
395
|
+
)
|
|
396
|
+
.slice(0, topK);
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* Filter candidates against locally appended rows only. Hydrated metadata has
|
|
401
|
+
* no body and therefore never participates in lexical detection.
|
|
402
|
+
*/
|
|
403
|
+
function filterIndexedNearDuplicates(candidates, indexState, opts = {}) {
|
|
404
|
+
const input = Array.isArray(candidates) ? candidates : [];
|
|
405
|
+
if (!indexState || !indexState.usable) {
|
|
406
|
+
return { kept: input.slice(), dropped: [], disabled: true };
|
|
407
|
+
}
|
|
408
|
+
const localRows = indexState.rows
|
|
409
|
+
.filter((row) => typeof row.body === 'string' && row.body.trim())
|
|
410
|
+
.map((row, index) => ({
|
|
411
|
+
id: row.learning_id || `local-index-${index}`,
|
|
412
|
+
title: row.title,
|
|
413
|
+
body: row.body,
|
|
414
|
+
category: row.category,
|
|
415
|
+
status: row.status || null,
|
|
416
|
+
}));
|
|
417
|
+
if (!localRows.length) return { kept: input.slice(), dropped: [], disabled: false };
|
|
418
|
+
|
|
419
|
+
try {
|
|
420
|
+
const kept = [];
|
|
421
|
+
const dropped = [];
|
|
422
|
+
for (const candidate of input) {
|
|
423
|
+
const result = findNearDuplicate(candidate, localRows);
|
|
424
|
+
if (result.verdict === 'flag') {
|
|
425
|
+
const matchedRow = localRows.find((row) => row.id === result.match.id);
|
|
426
|
+
dropped.push({
|
|
427
|
+
candidate,
|
|
428
|
+
match: {
|
|
429
|
+
...result.match,
|
|
430
|
+
title: matchedRow ? matchedRow.title : null,
|
|
431
|
+
},
|
|
432
|
+
});
|
|
433
|
+
} else {
|
|
434
|
+
kept.push(candidate);
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
return { kept, dropped, disabled: false };
|
|
438
|
+
} catch (error) {
|
|
439
|
+
loud(opts.log, `lexical dedup failed; keeping all extracted candidates: ${error.message}`);
|
|
440
|
+
return { kept: input.slice(), dropped: [], disabled: true };
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
module.exports = {
|
|
445
|
+
DEFAULT_INDEX_PATH,
|
|
446
|
+
HYDRATION_PAGE_SIZE,
|
|
447
|
+
PROMPT_MEMORY_MAX_TOKENS,
|
|
448
|
+
PROMPT_MEMORY_MAX_ROWS,
|
|
449
|
+
MEMORY_INSTRUCTIONS,
|
|
450
|
+
readExtractionIndex,
|
|
451
|
+
appendSubmittedLearning,
|
|
452
|
+
hydrateExtractionIndex,
|
|
453
|
+
loadIndexForExtraction,
|
|
454
|
+
buildPromptMemory,
|
|
455
|
+
estimatePromptTokens,
|
|
456
|
+
inferCategoryHints,
|
|
457
|
+
filterIndexedNearDuplicates,
|
|
458
|
+
rankIndexRowsForCandidate,
|
|
459
|
+
indexRowId,
|
|
460
|
+
localIndexRow,
|
|
461
|
+
hydratedIndexRow,
|
|
462
|
+
};
|
package/lib/installer.js
CHANGED
|
@@ -73,6 +73,9 @@ const RUNNER_STACK = Object.freeze([
|
|
|
73
73
|
['scripts/review-notice.js', 'scripts/review-notice.js', 0o755],
|
|
74
74
|
...sourceAdapterRows(),
|
|
75
75
|
['lib/sensitivity-filter.js', 'lib/sensitivity-filter.js', 0o644],
|
|
76
|
+
['lib/extraction-index.js', 'lib/extraction-index.js', 0o644],
|
|
77
|
+
['lib/similarity.js', 'lib/similarity.js', 0o644],
|
|
78
|
+
['config/near-duplicate.json', 'config/near-duplicate.json', 0o644],
|
|
76
79
|
]);
|
|
77
80
|
|
|
78
81
|
// ─── Client registry + detection (spec §LW-12 step 1; UC-0 expansion) ───────
|
|
@@ -18,7 +18,9 @@
|
|
|
18
18
|
// ─── Version ─────────────────────────────────────────────────────────────────
|
|
19
19
|
// Bumped when patterns are added or behavior changes. Server rejects extractions
|
|
20
20
|
// from clients older than N-1 (§7.6).
|
|
21
|
-
|
|
21
|
+
// 0.5.0 (task-#19, 2026-07-19): Google Drive/Docs/Sheets/Slides file-ID
|
|
22
|
+
// patterns — a private Drive doc ID survived the scrubber in the wild.
|
|
23
|
+
const SENSITIVITY_FILTER_VERSION = '0.5.0';
|
|
22
24
|
|
|
23
25
|
// ─── Patterns ────────────────────────────────────────────────────────────────
|
|
24
26
|
// Each pattern has a name, regex, and description for the rejection message.
|
|
@@ -165,6 +167,40 @@ const PATTERNS = [
|
|
|
165
167
|
regex: /[MN][A-Za-z\d]{23}\.[\w-]{6}\.[\w-]{27}/g,
|
|
166
168
|
description: 'Discord bot token',
|
|
167
169
|
},
|
|
170
|
+
// ── task-#19 (2026-07-19): Google Drive file IDs ──────────────────────────
|
|
171
|
+
// Found live: a private Google Drive doc ID survived the scrubber. A Drive
|
|
172
|
+
// ID is a capability reference — a link-shared doc is readable by ANYONE
|
|
173
|
+
// holding the ID — so it is credentials-class, not merely PII-class.
|
|
174
|
+
{
|
|
175
|
+
// The /d/<id>/ URL shapes across the editors + Drive, plus the id= and
|
|
176
|
+
// folders/ forms. Precise (requires the google.com host shape), so it
|
|
177
|
+
// fires unconditionally — a Drive URL in a public learning is never a
|
|
178
|
+
// false positive worth waving through.
|
|
179
|
+
name: 'google_drive_url',
|
|
180
|
+
regex: /(?:docs|drive|sheets|slides)\.google\.com\/(?:(?:document|spreadsheets|presentation|forms|drawings|file)\/(?:u\/\d+\/)?d\/[A-Za-z0-9_-]{20,}|open\?[^\s]*\bid=[A-Za-z0-9_-]{20,}|uc\?[^\s]*\bid=[A-Za-z0-9_-]{20,}|drive\/(?:u\/\d+\/)?folders\/[A-Za-z0-9_-]{20,}|folderview\?[^\s]*\bid=[A-Za-z0-9_-]{20,})/g,
|
|
181
|
+
description: 'Google Drive/Docs/Sheets/Slides URL exposing a file ID',
|
|
182
|
+
},
|
|
183
|
+
{
|
|
184
|
+
// Bare Drive-ID heuristic, GUARDED (this filter is a hard 422 at /learn,
|
|
185
|
+
// so a loose rule bounces legit content). Shape: modern IDs start with
|
|
186
|
+
// '1' (33/44 chars), legacy folder IDs with '0B'; charset base64url.
|
|
187
|
+
// Gate-A F3: the validate hook requires MIXED CASE + a digit beyond the
|
|
188
|
+
// prefix char. The original uppercase/-/_ guard still 422'd six real FP
|
|
189
|
+
// classes the reviewer probed: issue-numbered branch names
|
|
190
|
+
// (1234-fix-the-thing…), numeric-leading URL slugs (10-ways-to-…),
|
|
191
|
+
// digit-1-leading UUIDs (~6% of v4s), numeric-separator literals
|
|
192
|
+
// (1_000_000_…), kebab date-ranges (19-07-2026-to-…), and UPPERCASE
|
|
193
|
+
// 40-hex SHAs. Mixed-case+digit excludes every one (branch/slug/UUID/
|
|
194
|
+
// date-range have no uppercase; separator literals no lowercase;
|
|
195
|
+
// uppercase SHAs no lowercase) while keeping real Drive IDs: for 32
|
|
196
|
+
// random base64url chars P(no lowercase)≈P(no uppercase)≈6e-8 and
|
|
197
|
+
// P(no digit)≈(54/64)^32≈0.4% — the URL rule above still catches every
|
|
198
|
+
// linked form regardless.
|
|
199
|
+
name: 'google_drive_id',
|
|
200
|
+
regex: /(?<![A-Za-z0-9_-])(?:1|0B)[A-Za-z0-9_-]{24,63}(?![A-Za-z0-9_-])/g,
|
|
201
|
+
description: 'Bare Google Drive file ID (25+ char base64url, 1/0B prefix)',
|
|
202
|
+
validate: (m) => /[A-Z]/.test(m) && /[a-z]/.test(m) && /\d/.test(m.slice(1)),
|
|
203
|
+
},
|
|
168
204
|
];
|
|
169
205
|
|
|
170
206
|
// ─── M-2: /g flag invariant assertion at module load ────────────────────────
|
|
@@ -220,6 +256,11 @@ function scanLearning(learning) {
|
|
|
220
256
|
if (m[0].length <= 42) continue;
|
|
221
257
|
}
|
|
222
258
|
|
|
259
|
+
// task-#19: optional per-pattern validate hook — a match the hook
|
|
260
|
+
// refuses is skipped (used to guard heuristic patterns whose raw
|
|
261
|
+
// regex would over-fire, e.g. the bare Drive-ID rule).
|
|
262
|
+
if (pattern.validate && !pattern.validate(m[0])) continue;
|
|
263
|
+
|
|
223
264
|
matches.push({
|
|
224
265
|
pattern: pattern.name,
|
|
225
266
|
field: fieldName,
|
|
@@ -280,6 +321,9 @@ function getRedactionHint(patternName) {
|
|
|
280
321
|
openai_project_key: '{OPENAI_KEY}',
|
|
281
322
|
anthropic_key: '{ANTHROPIC_KEY}',
|
|
282
323
|
discord_bot_token: '{DISCORD_TOKEN}',
|
|
324
|
+
// task-#19 patterns
|
|
325
|
+
google_drive_url: 'https://docs.google.com/document/d/{DRIVE_FILE_ID}/',
|
|
326
|
+
google_drive_id: '{DRIVE_FILE_ID}',
|
|
283
327
|
};
|
|
284
328
|
return hints[patternName] || '{REDACTED}';
|
|
285
329
|
}
|
|
@@ -345,6 +389,9 @@ function scanText(text) {
|
|
|
345
389
|
// Private key length check (same as scanLearning)
|
|
346
390
|
if (pattern.name === 'private_key' && m[0].length <= 42) continue;
|
|
347
391
|
|
|
392
|
+
// task-#19: per-pattern validate hook (same semantics as scanLearning).
|
|
393
|
+
if (pattern.validate && !pattern.validate(m[0])) continue;
|
|
394
|
+
|
|
348
395
|
matchList.push({
|
|
349
396
|
pattern: pattern.name,
|
|
350
397
|
match: redactMatch(m[0]),
|