advisers 0.1.0-preview.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +131 -21
- package/NOTICE +34 -0
- package/README-zh.md +40 -9
- package/README.md +53 -10
- package/THIRD-PARTY-NOTICES.md +277 -0
- package/bin/advisers.js +4230 -122
- package/index.js +1 -1
- package/lib/agent.js +414 -0
- package/lib/attachments.js +283 -0
- package/lib/capture.js +168 -0
- package/lib/context-window.js +123 -0
- package/lib/digest.js +27 -0
- package/lib/doc-review.js +30 -0
- package/lib/egress.js +27 -0
- package/lib/evidence.js +1995 -0
- package/lib/ical.js +542 -0
- package/lib/imap.js +181 -0
- package/lib/importers.js +1001 -0
- package/lib/learning.js +46 -0
- package/lib/mcp-server.js +869 -0
- package/lib/model.js +307 -0
- package/lib/notify.js +189 -0
- package/lib/nudge.js +10 -0
- package/lib/ocr.js +47 -0
- package/lib/pdf.js +105 -0
- package/lib/pet-brief-store.js +51 -0
- package/lib/pet-brief.js +644 -0
- package/lib/receipt-verify.js +624 -0
- package/lib/relative-date.js +393 -0
- package/lib/remind.js +690 -0
- package/lib/research.js +193 -0
- package/lib/review.js +88 -0
- package/lib/routine.js +22 -0
- package/lib/safety/classifier.js +392 -0
- package/lib/safety/explain.js +157 -0
- package/lib/safety/index.js +333 -0
- package/lib/safety/redact.js +212 -0
- package/lib/safety/rules.js +824 -0
- package/lib/scheduler.js +399 -0
- package/lib/source.js +294 -0
- package/lib/sqlite.js +1022 -0
- package/lib/sync.js +129 -0
- package/lib/timestamp.js +45 -0
- package/lib/tos/index.js +332 -0
- package/lib/tos/manifest.json +32 -0
- package/lib/tos/privacy.zh-CN.md +136 -0
- package/lib/tos/safety.zh-CN.md +101 -0
- package/lib/tos/summary.zh-CN.md +12 -0
- package/lib/tos/terms.zh-CN.md +439 -0
- package/lib/ui.js +49 -0
- package/lib/undo.js +83 -0
- package/lib/vault.js +51 -0
- package/package.json +26 -4
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
import { readFile, stat } from 'node:fs/promises'
|
|
2
|
+
import { existsSync, lstatSync, realpathSync } from 'node:fs'
|
|
3
|
+
import { extname, isAbsolute, relative, resolve } from 'node:path'
|
|
4
|
+
import JSZip from 'jszip'
|
|
5
|
+
import { runLocalOcr, ocrConfigured } from './ocr.js'
|
|
6
|
+
import { classify, safetyErrorFrom } from './safety/index.js'
|
|
7
|
+
|
|
8
|
+
const MAX_FILE_BYTES = 10 * 1024 * 1024
|
|
9
|
+
const MAX_CHARS = 20_000
|
|
10
|
+
// ZIP extraction budget: keep the same limits as advisers-office read.ts
|
|
11
|
+
// (DEFAULT_ZIP_BUDGET) so an attachment is rejected, not decompressed, when a
|
|
12
|
+
// malicious docx/xlsx/pptx/ofd would expand past them.
|
|
13
|
+
const MAX_ZIP_ENTRIES = 2000
|
|
14
|
+
const MAX_TOTAL_UNCOMPRESSED = 64 * 1024 * 1024
|
|
15
|
+
const MAX_ENTRY_UNCOMPRESSED = 16 * 1024 * 1024
|
|
16
|
+
const MAX_TEXT_CHARS = 8 * 1024 * 1024
|
|
17
|
+
const TEXT_EXT = new Set(['.txt', '.md', '.markdown', '.json', '.csv', '.log', '.yaml', '.yml', '.xml', '.html'])
|
|
18
|
+
const OFFICE_EXT = new Set(['.docx', '.xlsx', '.pptx'])
|
|
19
|
+
const IMAGE_EXT = new Set(['.png', '.jpg', '.jpeg', '.webp', '.bmp', '.tif', '.tiff'])
|
|
20
|
+
|
|
21
|
+
// Content-safety wiring (docs/CONTENT-SAFETY.md section 8). Attachments are the
|
|
22
|
+
// user's own files, so the classifier policy downgrades ordinary non-CSAM blocks
|
|
23
|
+
// to confirm; CSAM is a hard block and throws a typed, text-free error. The
|
|
24
|
+
// returned safety object is additive metadata (never the extracted text).
|
|
25
|
+
const ATTACHMENT_SURFACE = 'attachment' // alias of attachments.text
|
|
26
|
+
|
|
27
|
+
function attachmentSafety(result) {
|
|
28
|
+
return {
|
|
29
|
+
action: result.action,
|
|
30
|
+
code: result.code,
|
|
31
|
+
categories: Array.isArray(result.categories) ? [...result.categories] : [],
|
|
32
|
+
ruleIds: Array.isArray(result.ruleIds) ? [...result.ruleIds] : [],
|
|
33
|
+
severity: result.severity,
|
|
34
|
+
crisis: Boolean(result.crisis),
|
|
35
|
+
downgraded: Boolean(result.downgraded),
|
|
36
|
+
rulesetVersion: result.rulesetVersion,
|
|
37
|
+
evidenceHash: result.evidenceHash || null,
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Classify extracted text; throw on a hard block, otherwise attach metadata. */
|
|
42
|
+
function withSafety(extracted, options = {}) {
|
|
43
|
+
const safety = classify(extracted.text, { surface: ATTACHMENT_SURFACE, hmacKey: options.hmacKey })
|
|
44
|
+
if (safety.action === 'block') throw safetyErrorFrom(safety, { surface: ATTACHMENT_SURFACE })
|
|
45
|
+
return { ...extracted, safety: attachmentSafety(safety) }
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function safePath(cwd, input) {
|
|
49
|
+
const root = realpathSync(cwd)
|
|
50
|
+
const abs = resolve(root, input)
|
|
51
|
+
const rel = relative(root, abs)
|
|
52
|
+
if (rel.startsWith('..') || isAbsolute(rel)) throw new Error(`attachment escapes workspace: ${input}`)
|
|
53
|
+
const parts = rel.split(/[\\/]+/).filter(Boolean)
|
|
54
|
+
let current = root
|
|
55
|
+
for (const part of parts) {
|
|
56
|
+
current = resolve(current, part)
|
|
57
|
+
if (existsSync(current) && lstatSync(current).isSymbolicLink()) throw new Error(`attachment symlink is not allowed: ${part}`)
|
|
58
|
+
}
|
|
59
|
+
return abs
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function decodeXml(text) {
|
|
63
|
+
return text
|
|
64
|
+
.replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"').replace(/'/g, "'").replace(/&/g, '&')
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function truncate(text) {
|
|
68
|
+
if (text.length <= MAX_CHARS) return { text, truncated: false }
|
|
69
|
+
return { text: text.slice(0, MAX_CHARS), truncated: true }
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Declared uncompressed size from the ZIP central directory (never byte 0). */
|
|
73
|
+
function declaredUncompressedSize(entry) {
|
|
74
|
+
const size = entry?._data?.uncompressedSize
|
|
75
|
+
return typeof size === 'number' && Number.isFinite(size) && size >= 0 ? size : undefined
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Bounded reader over one ZIP archive. Every entry is decompressed as a capped
|
|
80
|
+
* stream, so a lying central directory cannot inflate past the budget, and the
|
|
81
|
+
* cumulative decompressed bytes stay inside the total budget.
|
|
82
|
+
*/
|
|
83
|
+
class ZipBudgetReader {
|
|
84
|
+
constructor(zip) {
|
|
85
|
+
this.zip = zip
|
|
86
|
+
this.decompressedBytes = 0
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
names() {
|
|
90
|
+
return Object.keys(this.zip.files)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async readText(name) {
|
|
94
|
+
const entry = this.zip.files[name]
|
|
95
|
+
if (!entry || entry.dir) return undefined
|
|
96
|
+
const buffer = await this.readBuffer(name)
|
|
97
|
+
const text = buffer.toString('utf8')
|
|
98
|
+
if (text.length > MAX_TEXT_CHARS) {
|
|
99
|
+
throw new Error(`attachment text entry too large: ${name} decodes to ${text.length} characters (limit ${MAX_TEXT_CHARS})`)
|
|
100
|
+
}
|
|
101
|
+
return text
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async readBuffer(name) {
|
|
105
|
+
const entry = this.zip.files[name]
|
|
106
|
+
if (!entry || entry.dir) throw new Error(`attachment ZIP entry is missing: ${name}`)
|
|
107
|
+
const declared = declaredUncompressedSize(entry)
|
|
108
|
+
if (declared !== undefined && declared > MAX_ENTRY_UNCOMPRESSED) {
|
|
109
|
+
throw new Error(`attachment entry too large: ${name} declares ${declared} uncompressed bytes (limit ${MAX_ENTRY_UNCOMPRESSED})`)
|
|
110
|
+
}
|
|
111
|
+
const stream = entry.nodeStream('nodebuffer')
|
|
112
|
+
const chunks = []
|
|
113
|
+
const self = this
|
|
114
|
+
let entryBytes = 0
|
|
115
|
+
await new Promise((resolve, reject) => {
|
|
116
|
+
let settled = false
|
|
117
|
+
const fail = (error) => {
|
|
118
|
+
if (settled) return
|
|
119
|
+
settled = true
|
|
120
|
+
reject(error)
|
|
121
|
+
}
|
|
122
|
+
const stop = () => {
|
|
123
|
+
if (typeof stream.destroy === 'function') stream.destroy()
|
|
124
|
+
}
|
|
125
|
+
stream.on('data', (chunk) => {
|
|
126
|
+
if (settled) return
|
|
127
|
+
entryBytes += chunk.length
|
|
128
|
+
self.decompressedBytes += chunk.length
|
|
129
|
+
if (entryBytes > MAX_ENTRY_UNCOMPRESSED) {
|
|
130
|
+
fail(new Error(`attachment entry expands beyond the single-entry budget: ${name} exceeds ${MAX_ENTRY_UNCOMPRESSED} uncompressed bytes`))
|
|
131
|
+
stop()
|
|
132
|
+
return
|
|
133
|
+
}
|
|
134
|
+
if (self.decompressedBytes > MAX_TOTAL_UNCOMPRESSED) {
|
|
135
|
+
fail(new Error(`attachment archive expands beyond the unzip budget: more than ${MAX_TOTAL_UNCOMPRESSED} uncompressed bytes`))
|
|
136
|
+
stop()
|
|
137
|
+
return
|
|
138
|
+
}
|
|
139
|
+
chunks.push(chunk)
|
|
140
|
+
})
|
|
141
|
+
stream.on('end', () => {
|
|
142
|
+
if (settled) return
|
|
143
|
+
settled = true
|
|
144
|
+
resolve()
|
|
145
|
+
})
|
|
146
|
+
stream.on('error', (error) => fail(error))
|
|
147
|
+
})
|
|
148
|
+
return Buffer.concat(chunks, entryBytes)
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** Reject an archive before any entry is decompressed. */
|
|
153
|
+
async function loadZip(buffer) {
|
|
154
|
+
const zip = await JSZip.loadAsync(buffer)
|
|
155
|
+
const names = Object.keys(zip.files)
|
|
156
|
+
if (names.length > MAX_ZIP_ENTRIES) {
|
|
157
|
+
throw new Error(`attachment archive has too many ZIP entries: ${names.length} (limit ${MAX_ZIP_ENTRIES})`)
|
|
158
|
+
}
|
|
159
|
+
let declaredTotal = 0
|
|
160
|
+
for (const name of names) {
|
|
161
|
+
const entry = zip.files[name]
|
|
162
|
+
if (!entry || entry.dir) continue
|
|
163
|
+
const declared = declaredUncompressedSize(entry) ?? 0
|
|
164
|
+
if (declared > MAX_ENTRY_UNCOMPRESSED) {
|
|
165
|
+
throw new Error(`attachment entry too large: ${name} declares ${declared} uncompressed bytes (limit ${MAX_ENTRY_UNCOMPRESSED})`)
|
|
166
|
+
}
|
|
167
|
+
declaredTotal += declared
|
|
168
|
+
if (declaredTotal > MAX_TOTAL_UNCOMPRESSED) {
|
|
169
|
+
throw new Error(`attachment archive expands beyond the unzip budget: declares ${declaredTotal} uncompressed bytes (limit ${MAX_TOTAL_UNCOMPRESSED})`)
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
return new ZipBudgetReader(zip)
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function docxText(xml) {
|
|
176
|
+
return decodeXml(xml
|
|
177
|
+
.replace(/<w:tab[^>]*\/>/g, '\t')
|
|
178
|
+
.replace(/<\/w:p>/g, '\n')
|
|
179
|
+
.replace(/<w:br[^>]*\/>/g, '\n')
|
|
180
|
+
.replace(/<[^>]+>/g, ''))
|
|
181
|
+
.replace(/\n{3,}/g, '\n\n').trim()
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
async function pptxText(reader) {
|
|
185
|
+
const slides = reader.names().filter((n) => /^ppt\/slides\/slide\d+\.xml$/.test(n)).sort((a, b) => a.localeCompare(b, undefined, { numeric: true }))
|
|
186
|
+
const out = []
|
|
187
|
+
for (const name of slides) {
|
|
188
|
+
const xml = await reader.readText(name)
|
|
189
|
+
if (!xml) continue
|
|
190
|
+
const texts = [...xml.matchAll(/<a:t>([\s\S]*?)<\/a:t>/g)].map((m) => decodeXml(m[1]))
|
|
191
|
+
if (texts.length) out.push(`# ${name.split('/').pop()}\n${texts.join('\n')}`)
|
|
192
|
+
}
|
|
193
|
+
return out.join('\n\n').trim()
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
export async function ofdText(buffer) {
|
|
197
|
+
const reader = await loadZip(buffer)
|
|
198
|
+
const names = reader.names().filter((n) => /(?:^|\/)Content\.xml$/i.test(n) || /(?:^|\/)Pages\/[^/]+\.xml$/i.test(n)).sort()
|
|
199
|
+
const out = []
|
|
200
|
+
for (const name of names) {
|
|
201
|
+
const xml = await reader.readText(name)
|
|
202
|
+
if (!xml) continue
|
|
203
|
+
const texts = [...xml.matchAll(/<(?:ofd:)?TextCode[^>]*>([\s\S]*?)<\/(?:ofd:)?TextCode>/gi)].map((m) => decodeXml(m[1]))
|
|
204
|
+
if (texts.length) out.push(texts.join('\n'))
|
|
205
|
+
}
|
|
206
|
+
return out.join('\n\n').replace(/\n{3,}/g, '\n\n').trim()
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
async function pdfText(path) {
|
|
210
|
+
const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs')
|
|
211
|
+
const data = new Uint8Array(await readFile(path))
|
|
212
|
+
const task = pdfjs.getDocument({ data, isEvalSupported: false, useSystemFonts: false, disableFontFace: true, verbosity: 0 })
|
|
213
|
+
const doc = await task.promise
|
|
214
|
+
try {
|
|
215
|
+
const pages = []
|
|
216
|
+
const count = Math.min(doc.numPages, 50)
|
|
217
|
+
for (let pageNo = 1; pageNo <= count; pageNo++) {
|
|
218
|
+
const page = await doc.getPage(pageNo)
|
|
219
|
+
const content = await page.getTextContent()
|
|
220
|
+
pages.push(content.items.map((item) => item.str ?? '').join(' '))
|
|
221
|
+
}
|
|
222
|
+
return pages.join('\n').trim()
|
|
223
|
+
} finally {
|
|
224
|
+
await task.destroy()
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
async function xlsxText(reader) {
|
|
229
|
+
const out = []
|
|
230
|
+
const shared = await reader.readText('xl/sharedStrings.xml')
|
|
231
|
+
if (shared) out.push(...[...shared.matchAll(/<t[^>]*>([\s\S]*?)<\/t>/g)].map((m) => decodeXml(m[1])))
|
|
232
|
+
if (!out.length) {
|
|
233
|
+
for (const name of reader.names().filter((n) => /^xl\/worksheets\/sheet\d+\.xml$/.test(n))) {
|
|
234
|
+
const xml = await reader.readText(name)
|
|
235
|
+
if (xml) out.push(...[...xml.matchAll(/<t[^>]*>([\s\S]*?)<\/t>/g)].map((m) => decodeXml(m[1])))
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
return out.filter(Boolean).join('\n').trim()
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
export async function extractAttachment(input, { cwd = process.cwd(), hmacKey } = {}) {
|
|
242
|
+
const path = safePath(cwd, input)
|
|
243
|
+
const info = await stat(path)
|
|
244
|
+
if (!info.isFile()) throw new Error(`not a file: ${input}`)
|
|
245
|
+
if (info.size > MAX_FILE_BYTES) throw new Error(`attachment is larger than 10 MB: ${input}`)
|
|
246
|
+
const ext = extname(path).toLowerCase()
|
|
247
|
+
const name = path.split('/').pop()
|
|
248
|
+
if (TEXT_EXT.has(ext)) {
|
|
249
|
+
const raw = await readFile(path, 'utf8')
|
|
250
|
+
const { text, truncated } = truncate(raw)
|
|
251
|
+
return withSafety({ name, path, kind: ext.slice(1), text, truncated, bytes: info.size }, { hmacKey })
|
|
252
|
+
}
|
|
253
|
+
if (OFFICE_EXT.has(ext)) {
|
|
254
|
+
const reader = await loadZip(await readFile(path))
|
|
255
|
+
const raw = ext === '.docx'
|
|
256
|
+
? docxText(await reader.readText('word/document.xml') ?? '')
|
|
257
|
+
: ext === '.pptx'
|
|
258
|
+
? await pptxText(reader)
|
|
259
|
+
: await xlsxText(reader)
|
|
260
|
+
const { text, truncated } = truncate(raw)
|
|
261
|
+
if (!text) throw new Error(`no extractable text in ${name}`)
|
|
262
|
+
return withSafety({ name, path, kind: ext.slice(1), text, truncated, bytes: info.size }, { hmacKey })
|
|
263
|
+
}
|
|
264
|
+
if (ext === '.ofd') {
|
|
265
|
+
const raw = await ofdText(await readFile(path))
|
|
266
|
+
const { text, truncated } = truncate(raw)
|
|
267
|
+
if (!text) throw new Error(`no extractable text in ${name} (OFD image-only?)`)
|
|
268
|
+
return withSafety({ name, path, kind: 'ofd', text, truncated, bytes: info.size }, { hmacKey })
|
|
269
|
+
}
|
|
270
|
+
if (IMAGE_EXT.has(ext)) {
|
|
271
|
+
if (!ocrConfigured()) throw new Error(`image OCR is not configured; set ADVISERS_OCR_CMD to a local OCR executable (${name})`)
|
|
272
|
+
const raw = await runLocalOcr(path, { cwd })
|
|
273
|
+
const { text, truncated } = truncate(raw)
|
|
274
|
+
return withSafety({ name, path, kind: 'ocr', text, truncated, bytes: info.size }, { hmacKey })
|
|
275
|
+
}
|
|
276
|
+
if (ext === '.pdf') {
|
|
277
|
+
const raw = await pdfText(path)
|
|
278
|
+
const { text, truncated } = truncate(raw)
|
|
279
|
+
if (!text) throw new Error(`no extractable text in ${name} (scanned PDF?)`)
|
|
280
|
+
return withSafety({ name, path, kind: 'pdf', text, truncated, bytes: info.size }, { hmacKey })
|
|
281
|
+
}
|
|
282
|
+
throw new Error(`unsupported attachment type: ${ext || '(none)'}`)
|
|
283
|
+
}
|
package/lib/capture.js
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import { chatCompletion } from './model.js'
|
|
2
|
+
import { describeMessageTime, normalizeDue } from './relative-date.js'
|
|
3
|
+
import { classify, redact, safetyErrorFrom, buildAuditReason } from './safety/index.js'
|
|
4
|
+
|
|
5
|
+
const MAX_CANDIDATES = 5
|
|
6
|
+
const MAX_CHARS = 20_000
|
|
7
|
+
const CAPTURE_SURFACE = 'capture.input'
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Shared capture safety gate (also reused by lib/mcp-server.js).
|
|
11
|
+
*
|
|
12
|
+
* Classifies text with the capture surface (local-file policy: non-CSAM blocks
|
|
13
|
+
* are downgraded to confirm; CSAM stays a hard block) and throws a typed,
|
|
14
|
+
* text-free ContentSafetyError on block. Callers must run this before any model
|
|
15
|
+
* call or candidate write so a blocked input never produces a candidate.
|
|
16
|
+
*/
|
|
17
|
+
export function inspectCaptureText(text, { surface = CAPTURE_SURFACE, hmacKey } = {}) {
|
|
18
|
+
const result = classify(text, { surface, hmacKey })
|
|
19
|
+
if (result.action === 'block') throw safetyErrorFrom(result, { surface })
|
|
20
|
+
return result
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function safetyWarningOf(result, direction) {
|
|
24
|
+
if (!result || result.action === 'allow') return null
|
|
25
|
+
const surface = result.surface || CAPTURE_SURFACE
|
|
26
|
+
const ruleIds = Array.isArray(result.ruleIds) ? [...result.ruleIds] : []
|
|
27
|
+
return {
|
|
28
|
+
direction,
|
|
29
|
+
action: result.action,
|
|
30
|
+
code: result.code,
|
|
31
|
+
categories: Array.isArray(result.categories) ? [...result.categories] : [],
|
|
32
|
+
ruleIds,
|
|
33
|
+
surface,
|
|
34
|
+
rulesetVersion: result.rulesetVersion,
|
|
35
|
+
evidenceHash: result.evidenceHash || null,
|
|
36
|
+
crisis: Boolean(result.crisis),
|
|
37
|
+
auditReason: buildAuditReason({
|
|
38
|
+
rulesetVersion: result.rulesetVersion,
|
|
39
|
+
surface,
|
|
40
|
+
action: result.action,
|
|
41
|
+
code: result.code,
|
|
42
|
+
ruleIds,
|
|
43
|
+
evidenceHash: result.evidenceHash,
|
|
44
|
+
}),
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Redact PII/secrets from the candidate payload only. The quote stays verbatim:
|
|
50
|
+
* it is checked against the source text, so redacting it would break the
|
|
51
|
+
* verbatim contract. Redaction is one-way and never returns the original value.
|
|
52
|
+
*/
|
|
53
|
+
function redactCandidatePayload(candidate) {
|
|
54
|
+
return {
|
|
55
|
+
...candidate,
|
|
56
|
+
action: redact(candidate.action).text,
|
|
57
|
+
uncertainty: redact(candidate.uncertainty).text,
|
|
58
|
+
...(candidate.creditor === undefined ? {} : { creditor: redact(candidate.creditor).text }),
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function normalize(text) {
|
|
63
|
+
return String(text || '').replace(/\r\n/g, '\n').replace(/[ \t]+/g, ' ').trim()
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function buildCapturePrompt(text, { date = new Date(), source = 'wechat' } = {}) {
|
|
67
|
+
const clock = describeMessageTime(date)
|
|
68
|
+
return [
|
|
69
|
+
'从下面这段中文工作内容里抽取“承诺/义务”候选。只做候选,不写账本。',
|
|
70
|
+
`来源:${source};消息时间:${clock.label};今天是 ${clock.date}(${clock.weekday})。`,
|
|
71
|
+
'规则:',
|
|
72
|
+
'1. 只抽明确义务:我答应别人、别人答应我、双方待定;不要把请求、客套、计划、愿望当承诺。',
|
|
73
|
+
'2. quote 必须逐字来自原文,不改标点/不补字;找不到原句就不要输出。',
|
|
74
|
+
'2b. action 必须是 12 字以内的动词短语,去掉“把/给+人名”,例如“发报价”而不是“把报价发给王工”。',
|
|
75
|
+
'3. modal: firm(我会/一定/周五前)、tentative(尽量/争取/应该/回头)、conditional(如果/前提)。',
|
|
76
|
+
'4. dueAt 用 ISO 8601(带 +08:00);相对日期一律以“消息时间”的 Asia/Shanghai 本地日期为基准换算,禁止按 UTC 日期;无法确定就 null,并在 uncertainty 写明。',
|
|
77
|
+
'4b. 参考:今天=消息当天;明天=+1;后天=+2;周X=本周或下一个周X;下周X=下一周;月底=当月最后一天;季度末=当季最后一天。',
|
|
78
|
+
'5. debtor: user(我)/ third(对方)/ unknown;creditor 是对方名字,未知写 null。',
|
|
79
|
+
'6. 最多 5 条;宁少勿多;每条的 uncertainty 写一个短原因。',
|
|
80
|
+
'输出严格 JSON:{"candidates":[{"action":"","debtor":"user|third|unknown","creditor":null,"dueAt":null,"quote":"","modal":"firm|tentative|conditional","confidence":0.0,"uncertainty":""}]}',
|
|
81
|
+
'',
|
|
82
|
+
'原文:',
|
|
83
|
+
text,
|
|
84
|
+
].join('\n')
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export function parseCaptureResponse(content) {
|
|
88
|
+
const raw = String(content || '').trim().replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '')
|
|
89
|
+
let parsed
|
|
90
|
+
try { parsed = JSON.parse(raw) } catch { return [] }
|
|
91
|
+
if (!Array.isArray(parsed?.candidates)) return []
|
|
92
|
+
return parsed.candidates
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export function validateCaptureCandidates(candidates, text, { now = new Date(), messageDate, max = MAX_CANDIDATES } = {}) {
|
|
96
|
+
const original = normalize(text)
|
|
97
|
+
const compact = original.replace(/\s+/g, '')
|
|
98
|
+
const anchor = messageDate ?? now
|
|
99
|
+
const out = []
|
|
100
|
+
for (const c of Array.isArray(candidates) ? candidates : []) {
|
|
101
|
+
if (!c || typeof c !== 'object') continue
|
|
102
|
+
const quote = String(c.quote || '')
|
|
103
|
+
if (!quote || !compact.includes(quote.replace(/\s+/g, ''))) continue
|
|
104
|
+
const action = String(c.action || '').trim()
|
|
105
|
+
if (!action || action.length > 200) continue
|
|
106
|
+
const debtor = ['user', 'third', 'unknown'].includes(c.debtor) ? c.debtor : 'unknown'
|
|
107
|
+
// Deterministic date pass: the model is good at spotting commitments but
|
|
108
|
+
// unreliable at calendar arithmetic (and used to receive a UTC clock).
|
|
109
|
+
const due = normalizeDue({ dueAt: c.dueAt, quote, text: original, messageDate: anchor })
|
|
110
|
+
const ambiguityNote = due.ambiguous ? '相对日期有歧义,已按消息时间规则解析' : ''
|
|
111
|
+
out.push({
|
|
112
|
+
action,
|
|
113
|
+
debtor,
|
|
114
|
+
creditor: c.creditor == null ? undefined : String(c.creditor).slice(0, 60),
|
|
115
|
+
dueAt: due.dueAt,
|
|
116
|
+
dueBasis: due.basis,
|
|
117
|
+
dueAmbiguous: due.ambiguous || undefined,
|
|
118
|
+
quote,
|
|
119
|
+
modal: ['firm', 'tentative', 'conditional'].includes(c.modal) ? c.modal : 'tentative',
|
|
120
|
+
confidence: Math.max(0, Math.min(1, Number(c.confidence) || 0.5)),
|
|
121
|
+
uncertainty: String(c.uncertainty || ambiguityNote || '').slice(0, 160),
|
|
122
|
+
})
|
|
123
|
+
if (out.length >= max) break
|
|
124
|
+
}
|
|
125
|
+
return out
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export async function captureCandidates({ text, apiKey, baseUrl, model, fetchImpl = fetch, date, source, now = new Date(), safetyKey }) {
|
|
129
|
+
const input = normalize(text)
|
|
130
|
+
if (!input) return { candidates: [], usage: undefined }
|
|
131
|
+
if (input.length > MAX_CHARS) throw new Error(`capture input is too long (max ${MAX_CHARS} chars)`)
|
|
132
|
+
const messageDate = date ? new Date(date) : now
|
|
133
|
+
if (!Number.isFinite(messageDate.getTime())) throw new Error('capture message date is invalid')
|
|
134
|
+
// Fail closed before the model call: a blocked input produces no candidate.
|
|
135
|
+
const inputSafety = inspectCaptureText(input, { hmacKey: safetyKey })
|
|
136
|
+
const safetyWarnings = []
|
|
137
|
+
const inputWarning = safetyWarningOf(inputSafety, 'input')
|
|
138
|
+
if (inputWarning) safetyWarnings.push(inputWarning)
|
|
139
|
+
const result = await chatCompletion({
|
|
140
|
+
messages: [
|
|
141
|
+
{ role: 'system', content: '你是严谨的中文工作秘书。只输出 JSON,不输出解释。' },
|
|
142
|
+
{ role: 'user', content: buildCapturePrompt(input, { date: messageDate, source }) },
|
|
143
|
+
],
|
|
144
|
+
apiKey, baseUrl, model, fetchImpl,
|
|
145
|
+
task: 'capture', thinking: true, effort: 'high', maxTokens: 4096,
|
|
146
|
+
safetyKey,
|
|
147
|
+
})
|
|
148
|
+
const validated = validateCaptureCandidates(parseCaptureResponse(result.content), input, { now, messageDate })
|
|
149
|
+
const candidates = []
|
|
150
|
+
for (const candidate of validated) {
|
|
151
|
+
// A model-invented candidate that is itself block-level must not become a
|
|
152
|
+
// pending row; it is dropped (the batch continues) and only a structural
|
|
153
|
+
// warning is reported - never the original text.
|
|
154
|
+
const candidateSafety = classify(`${candidate.action}\n${candidate.quote}`, { surface: CAPTURE_SURFACE, hmacKey: safetyKey })
|
|
155
|
+
if (candidateSafety.action === 'block') {
|
|
156
|
+
const dropped = safetyWarningOf(candidateSafety, 'output')
|
|
157
|
+
if (dropped) safetyWarnings.push({ ...dropped, dropped: true })
|
|
158
|
+
continue
|
|
159
|
+
}
|
|
160
|
+
candidates.push(redactCandidatePayload(candidate))
|
|
161
|
+
}
|
|
162
|
+
return {
|
|
163
|
+
candidates,
|
|
164
|
+
usage: result.usage,
|
|
165
|
+
raw: result,
|
|
166
|
+
...(safetyWarnings.length ? { safetyWarnings } : {}),
|
|
167
|
+
}
|
|
168
|
+
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
export function estimateTokens(text) {
|
|
2
|
+
const value = String(text ?? '')
|
|
3
|
+
let cjk = 0
|
|
4
|
+
for (const ch of value) if (/[\u3400-\u9fff\uf900-\ufaff]/.test(ch)) cjk++
|
|
5
|
+
return Math.ceil(cjk + (value.length - cjk) / 4)
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export function shouldSummarize(turns, throughSeq = 0, every = 10, keepTurns = 20) {
|
|
9
|
+
const latest = turns.at(-1)?.seq ?? 0
|
|
10
|
+
return latest - keepTurns - throughSeq >= every
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/** Deterministic fallback; the caller may replace it with an LLM summary. */
|
|
14
|
+
export function summarizeTurns(turns, previous = '') {
|
|
15
|
+
const lines = []
|
|
16
|
+
if (previous) lines.push(previous.trim())
|
|
17
|
+
for (const turn of turns) {
|
|
18
|
+
const content = String(turn.content ?? '').replace(/\s+/g, ' ').trim()
|
|
19
|
+
if (!content) continue
|
|
20
|
+
const who = turn.role === 'user' ? '用户' : '助手'
|
|
21
|
+
lines.push(`- ${who}:${content.slice(0, 180)}`)
|
|
22
|
+
}
|
|
23
|
+
const text = lines.join('\n')
|
|
24
|
+
if (text.length <= 2400) return text
|
|
25
|
+
// Keep the head (pins/summary) and the latest lines; trim the middle, never the pins.
|
|
26
|
+
return `${text.slice(0, 1200)}\n…[summary trimmed]…\n${text.slice(-1000)}`
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Newest turns first backwards, but returned chronologically. */
|
|
30
|
+
export function selectTurns(turns, { maxTurns = 20, maxTokens = 3000 } = {}) {
|
|
31
|
+
const selected = []
|
|
32
|
+
let tokens = 0
|
|
33
|
+
for (let i = turns.length - 1; i >= 0; i--) {
|
|
34
|
+
const turn = turns[i]
|
|
35
|
+
if (selected.length >= maxTurns) break
|
|
36
|
+
const content = String(turn.content ?? '')
|
|
37
|
+
const capped = content.length > 4000 ? `${content.slice(0, 4000)}\n…[turn truncated]` : content
|
|
38
|
+
const cost = estimateTokens(capped) + 4
|
|
39
|
+
if (tokens + cost > maxTokens && selected.length > 0) break
|
|
40
|
+
selected.push({ ...turn, content: capped })
|
|
41
|
+
tokens += cost
|
|
42
|
+
}
|
|
43
|
+
return selected.reverse()
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function buildHistory({ turns = [], summary, throughSeq = 0, maxTurns = 20, maxTokens = 3000, keepTurns = 20 } = {}) {
|
|
47
|
+
const latest = turns.at(-1)?.seq ?? 0
|
|
48
|
+
// Never let compaction erase the live window: always retain the last keepTurns.
|
|
49
|
+
const cutoff = throughSeq >= latest ? Math.max(0, latest - keepTurns) : throughSeq
|
|
50
|
+
const fresh = turns.filter((t) => t.seq > cutoff)
|
|
51
|
+
const selected = selectTurns(fresh, { maxTurns, maxTokens })
|
|
52
|
+
return {
|
|
53
|
+
summary: summary?.trim() || '',
|
|
54
|
+
turns: selected,
|
|
55
|
+
usedTokens: estimateTokens(summary ?? '') + selected.reduce((n, t) => n + estimateTokens(t.content) + 4, 0),
|
|
56
|
+
dropped: fresh.length - selected.length,
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export function buildOpening({ summary, turns = [], commitments = [], now = new Date() } = {}) {
|
|
61
|
+
const lastUser = [...turns].reverse().find((t) => t.role === 'user')
|
|
62
|
+
const lastAssistant = [...turns].reverse().find((t) => t.role === 'assistant')
|
|
63
|
+
const due = commitments
|
|
64
|
+
.filter((c) => c.status === 'open' && c.dueAt)
|
|
65
|
+
.filter((c) => Date.parse(c.dueAt) <= now.getTime() + 24 * 3600_000)
|
|
66
|
+
.slice(0, 3)
|
|
67
|
+
const lines = []
|
|
68
|
+
if (summary) lines.push(`上次:${summary.replace(/\s+/g, ' ').slice(0, 160)}`)
|
|
69
|
+
else if (lastUser) lines.push(`上次你说:${lastUser.content.replace(/\s+/g, ' ').slice(0, 120)}`)
|
|
70
|
+
if (due.length) lines.push(`今天到期:${due.map((c) => `${c.action}${c.dueAt ? `(${c.dueAt.slice(0, 16)})` : ''}`).join(';')}`)
|
|
71
|
+
else lines.push('今天到期:无')
|
|
72
|
+
const action = due.length ? '先处理到期承诺,或告诉我今天先做什么' : (lastAssistant ? '继续上次的活,或告诉我今天先做什么' : '告诉我今天先做什么')
|
|
73
|
+
lines.push(`默认动作:${action}`)
|
|
74
|
+
return lines.join('\n')
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export function buildPins({ preferences = [], commitments = [], now = new Date() } = {}) {
|
|
78
|
+
const hard = preferences.filter((p) => p.priority === 'hard' && p.status === 'confirmed')
|
|
79
|
+
const due = commitments
|
|
80
|
+
.filter((c) => c.status === 'open' && c.dueAt && Date.parse(c.dueAt) <= now.getTime() + 14 * 86400_000)
|
|
81
|
+
.sort((a, b) => Date.parse(a.dueAt) - Date.parse(b.dueAt))
|
|
82
|
+
const lines = []
|
|
83
|
+
for (const p of hard.slice(0, 8)) lines.push(`hard: ${p.statement}`)
|
|
84
|
+
for (const c of due.slice(0, 6)) lines.push(`due: ${c.action}${c.dueAt ? ` (${c.dueAt})` : ''}`)
|
|
85
|
+
const text = lines.join('\n')
|
|
86
|
+
return text.length > 2000 ? `${text.slice(0, 2000)}\n…[pins truncated]` : text
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function continuityProbes({ summary = '', pins = '', turns = [] } = {}) {
|
|
90
|
+
const haystack = `${summary}\n${turns.map((t) => t.content).join('\n')}`
|
|
91
|
+
const missing = pins.split('\n').map((line) => line.trim()).filter((line) => line && !haystack.includes(line.replace(/^(hard|due):\s*/, '')))
|
|
92
|
+
return { ok: missing.length === 0, missing }
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export function ensurePins(summary, pins) {
|
|
96
|
+
if (!pins) return summary
|
|
97
|
+
const missing = pins.split('\n').filter((line) => line.trim() && !summary.includes(line.replace(/^(hard|due):\s*/, '')))
|
|
98
|
+
return missing.length ? `${pins}\n${summary}` : summary
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export class BudgetOverflowError extends Error {
|
|
102
|
+
constructor(used, max) {
|
|
103
|
+
super(`context budget exceeded: ${used} > ${max} estimated tokens`)
|
|
104
|
+
this.name = 'BudgetOverflowError'
|
|
105
|
+
this.used = used
|
|
106
|
+
this.max = max
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** Keep history inside a hard budget; never silently truncate the system/pins. */
|
|
111
|
+
export function fitHistory({ system = '', summary = '', history = [], current = '', maxTokens = 12000 } = {}) {
|
|
112
|
+
const kept = [...history]
|
|
113
|
+
const fixed = estimateTokens(system) + estimateTokens(summary) + estimateTokens(current)
|
|
114
|
+
let used = fixed + kept.reduce((n, t) => n + estimateTokens(t.content) + 4, 0)
|
|
115
|
+
let trimmed = 0
|
|
116
|
+
while (used > maxTokens && kept.length > 0) {
|
|
117
|
+
const removed = kept.shift()
|
|
118
|
+
used -= estimateTokens(removed.content) + 4
|
|
119
|
+
trimmed++
|
|
120
|
+
}
|
|
121
|
+
if (used > maxTokens) throw new BudgetOverflowError(used, maxTokens)
|
|
122
|
+
return { history: kept, usedTokens: used, trimmed }
|
|
123
|
+
}
|
package/lib/digest.js
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
function shanghaiMidnight(now) {
|
|
2
|
+
const parts = new Intl.DateTimeFormat('en-CA', { timeZone: 'Asia/Shanghai', year: 'numeric', month: '2-digit', day: '2-digit' }).format(now).split('-').map(Number)
|
|
3
|
+
return new Date(Date.UTC(parts[0], parts[1] - 1, parts[2], -8, 0, 0))
|
|
4
|
+
}
|
|
5
|
+
|
|
6
|
+
export function buildDigest({ period = 'day', now = new Date(), turns = [], commitments = [], deliveries = [] } = {}) {
|
|
7
|
+
const start = period === 'week'
|
|
8
|
+
? new Date(shanghaiMidnight(now).getTime() - 6 * 86400_000)
|
|
9
|
+
: shanghaiMidnight(now)
|
|
10
|
+
const inPeriod = (iso) => {
|
|
11
|
+
const t = Date.parse(iso)
|
|
12
|
+
return Number.isFinite(t) && t >= start.getTime() && t <= now.getTime()
|
|
13
|
+
}
|
|
14
|
+
const userTurns = turns.filter((t) => t.role === 'user' && inPeriod(t.createdAt))
|
|
15
|
+
const done = commitments.filter((c) => c.status === 'done' && c.doneAt && inPeriod(c.doneAt))
|
|
16
|
+
const open = commitments.filter((c) => c.status === 'open')
|
|
17
|
+
const due = open.filter((c) => c.dueAt && Date.parse(c.dueAt) <= now.getTime() + 86400_000)
|
|
18
|
+
const files = deliveries.filter((d) => inPeriod(d.createdAt))
|
|
19
|
+
const lines = [
|
|
20
|
+
`${period === 'week' ? '本周' : '今天'}:${userTurns.length} 次对话,${files.length} 个交付,${done.length} 个承诺完成。`,
|
|
21
|
+
]
|
|
22
|
+
if (due.length) lines.push(`到期:${due.slice(0, 3).map((c) => c.action).join(';')}`)
|
|
23
|
+
else lines.push('到期:无')
|
|
24
|
+
if (done.length) lines.push(`完成:${done.slice(0, 3).map((c) => c.action).join(';')}`)
|
|
25
|
+
if (userTurns.length) lines.push(`最近一句:${userTurns.at(-1).content.replace(/\s+/g, ' ').slice(0, 100)}`)
|
|
26
|
+
return lines.join('\n')
|
|
27
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { chatCompletion } from './model.js'
|
|
2
|
+
|
|
3
|
+
const MAX_INPUT = 20_000
|
|
4
|
+
|
|
5
|
+
export function parseDocReview(content) {
|
|
6
|
+
const raw = String(content || '').trim().replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '')
|
|
7
|
+
let parsed
|
|
8
|
+
try { parsed = JSON.parse(raw) } catch { return { findings: [], summary: '', invalid: true } }
|
|
9
|
+
const findings = Array.isArray(parsed?.findings) ? parsed.findings.slice(0, 8).map((f) => ({
|
|
10
|
+
severity: f.severity === 'blocker' ? 'blocker' : 'warn',
|
|
11
|
+
message: String(f.message || '').slice(0, 200),
|
|
12
|
+
evidence: String(f.evidence || '').slice(0, 160),
|
|
13
|
+
})).filter((f) => f.message) : []
|
|
14
|
+
return { findings, summary: String(parsed?.summary || '').slice(0, 300), invalid: false }
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export async function reviewDocument({ text, request, apiKey, baseUrl, model, fetchImpl = fetch }) {
|
|
18
|
+
const input = String(text || '').slice(0, MAX_INPUT)
|
|
19
|
+
if (!input.trim()) throw new Error('document has no readable text')
|
|
20
|
+
if (!apiKey) throw new Error('document review requires ADVISERS_API_KEY')
|
|
21
|
+
const response = await chatCompletion({
|
|
22
|
+
apiKey, baseUrl, model, fetchImpl,
|
|
23
|
+
task: 'review', thinking: true, effort: 'high', maxTokens: 4096,
|
|
24
|
+
messages: [
|
|
25
|
+
{ role: 'system', content: '你是交付文档的审阅者。只输出 JSON:{"findings":[{"severity":"blocker|warn","message":"","evidence":"原文短引"}],"summary":""}。只基于给定文本;不判断未提供的内容;没有证据不要报;最多 8 条;没有明显问题返回空 findings。' },
|
|
26
|
+
{ role: 'user', content: `用户要求:${request || '(未提供)'}\n\n文档:\n${input}` },
|
|
27
|
+
],
|
|
28
|
+
})
|
|
29
|
+
return { ...parseDocReview(response.content), usage: response.usage }
|
|
30
|
+
}
|
package/lib/egress.js
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Egress guard for matter-scoped model calls.
|
|
3
|
+
*
|
|
4
|
+
* Matter.secrecy is a numeric level `0 | 1 | 2 | 3` (packages/core/src/types.ts).
|
|
5
|
+
* The old research guard compared it to the string `'secret'`, which can never
|
|
6
|
+
* be true for the stored shape — a protected matter was silently allowed to
|
|
7
|
+
* egress. Until the 0–3 secrecy levels are productized we fail closed for any
|
|
8
|
+
* level above 0.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
const BLOCK_ABOVE = 0
|
|
12
|
+
|
|
13
|
+
export function secrecyLevel(matter) {
|
|
14
|
+
const raw = matter?.secrecy
|
|
15
|
+
if (raw === undefined || raw === null) return 0
|
|
16
|
+
const level = Number(raw)
|
|
17
|
+
return Number.isFinite(level) ? level : Number.POSITIVE_INFINITY
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function egressDeniedReason(matter, command = 'research') {
|
|
21
|
+
if (!matter) return `${command} is disabled: active matter not found; egress requires a normal matter`
|
|
22
|
+
const level = secrecyLevel(matter)
|
|
23
|
+
if (level > BLOCK_ABOVE) {
|
|
24
|
+
return `${command} is disabled for protected matters (secrecy=${Number.isFinite(level) ? level : 'invalid'}); egress requires a matter with secrecy 0`
|
|
25
|
+
}
|
|
26
|
+
return null
|
|
27
|
+
}
|