advisers 0.2.0-preview.71 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.js CHANGED
@@ -1,5 +1,5 @@
1
1
  export * from 'advisers-core'
2
2
 
3
- export const version = '0.2.0-preview.71'
3
+ export const version = '0.2.0'
4
4
  export const distribution = 'cordis'
5
5
  export const status = 'bootstrapping'
@@ -3,14 +3,48 @@ import { existsSync, lstatSync, realpathSync } from 'node:fs'
3
3
  import { extname, isAbsolute, relative, resolve } from 'node:path'
4
4
  import JSZip from 'jszip'
5
5
  import { runLocalOcr, ocrConfigured } from './ocr.js'
6
+ import { classify, safetyErrorFrom } from './safety/index.js'
6
7
 
7
8
  const MAX_FILE_BYTES = 10 * 1024 * 1024
8
- const MAX_UNZIP_BYTES = 20 * 1024 * 1024
9
9
  const MAX_CHARS = 20_000
10
+ // ZIP extraction budget: keep the same limits as advisers-office read.ts
11
+ // (DEFAULT_ZIP_BUDGET) so an attachment is rejected, not decompressed, when a
12
+ // malicious docx/xlsx/pptx/ofd would expand past them.
13
+ const MAX_ZIP_ENTRIES = 2000
14
+ const MAX_TOTAL_UNCOMPRESSED = 64 * 1024 * 1024
15
+ const MAX_ENTRY_UNCOMPRESSED = 16 * 1024 * 1024
16
+ const MAX_TEXT_CHARS = 8 * 1024 * 1024
10
17
  const TEXT_EXT = new Set(['.txt', '.md', '.markdown', '.json', '.csv', '.log', '.yaml', '.yml', '.xml', '.html'])
11
18
  const OFFICE_EXT = new Set(['.docx', '.xlsx', '.pptx'])
12
19
  const IMAGE_EXT = new Set(['.png', '.jpg', '.jpeg', '.webp', '.bmp', '.tif', '.tiff'])
13
20
 
21
+ // Content-safety wiring (docs/CONTENT-SAFETY.md section 8). Attachments are the
22
+ // user's own files, so the classifier policy downgrades ordinary non-CSAM blocks
23
+ // to confirm; CSAM is a hard block and throws a typed, text-free error. The
24
+ // returned safety object is additive metadata (never the extracted text).
25
+ const ATTACHMENT_SURFACE = 'attachment' // alias of attachments.text
26
+
27
+ function attachmentSafety(result) {
28
+ return {
29
+ action: result.action,
30
+ code: result.code,
31
+ categories: Array.isArray(result.categories) ? [...result.categories] : [],
32
+ ruleIds: Array.isArray(result.ruleIds) ? [...result.ruleIds] : [],
33
+ severity: result.severity,
34
+ crisis: Boolean(result.crisis),
35
+ downgraded: Boolean(result.downgraded),
36
+ rulesetVersion: result.rulesetVersion,
37
+ evidenceHash: result.evidenceHash || null,
38
+ }
39
+ }
40
+
41
+ /** Classify extracted text; throw on a hard block, otherwise attach metadata. */
42
+ function withSafety(extracted, options = {}) {
43
+ const safety = classify(extracted.text, { surface: ATTACHMENT_SURFACE, hmacKey: options.hmacKey })
44
+ if (safety.action === 'block') throw safetyErrorFrom(safety, { surface: ATTACHMENT_SURFACE })
45
+ return { ...extracted, safety: attachmentSafety(safety) }
46
+ }
47
+
14
48
  function safePath(cwd, input) {
15
49
  const root = realpathSync(cwd)
16
50
  const abs = resolve(root, input)
@@ -35,15 +69,107 @@ function truncate(text) {
35
69
  return { text: text.slice(0, MAX_CHARS), truncated: true }
36
70
  }
37
71
 
72
+ /** Declared uncompressed size from the ZIP central directory (never byte 0). */
73
+ function declaredUncompressedSize(entry) {
74
+ const size = entry?._data?.uncompressedSize
75
+ return typeof size === 'number' && Number.isFinite(size) && size >= 0 ? size : undefined
76
+ }
77
+
78
+ /**
79
+ * Bounded reader over one ZIP archive. Every entry is decompressed as a capped
80
+ * stream, so a lying central directory cannot inflate past the budget, and the
81
+ * cumulative decompressed bytes stay inside the total budget.
82
+ */
83
+ class ZipBudgetReader {
84
+ constructor(zip) {
85
+ this.zip = zip
86
+ this.decompressedBytes = 0
87
+ }
88
+
89
+ names() {
90
+ return Object.keys(this.zip.files)
91
+ }
92
+
93
+ async readText(name) {
94
+ const entry = this.zip.files[name]
95
+ if (!entry || entry.dir) return undefined
96
+ const buffer = await this.readBuffer(name)
97
+ const text = buffer.toString('utf8')
98
+ if (text.length > MAX_TEXT_CHARS) {
99
+ throw new Error(`attachment text entry too large: ${name} decodes to ${text.length} characters (limit ${MAX_TEXT_CHARS})`)
100
+ }
101
+ return text
102
+ }
103
+
104
+ async readBuffer(name) {
105
+ const entry = this.zip.files[name]
106
+ if (!entry || entry.dir) throw new Error(`attachment ZIP entry is missing: ${name}`)
107
+ const declared = declaredUncompressedSize(entry)
108
+ if (declared !== undefined && declared > MAX_ENTRY_UNCOMPRESSED) {
109
+ throw new Error(`attachment entry too large: ${name} declares ${declared} uncompressed bytes (limit ${MAX_ENTRY_UNCOMPRESSED})`)
110
+ }
111
+ const stream = entry.nodeStream('nodebuffer')
112
+ const chunks = []
113
+ const self = this
114
+ let entryBytes = 0
115
+ await new Promise((resolve, reject) => {
116
+ let settled = false
117
+ const fail = (error) => {
118
+ if (settled) return
119
+ settled = true
120
+ reject(error)
121
+ }
122
+ const stop = () => {
123
+ if (typeof stream.destroy === 'function') stream.destroy()
124
+ }
125
+ stream.on('data', (chunk) => {
126
+ if (settled) return
127
+ entryBytes += chunk.length
128
+ self.decompressedBytes += chunk.length
129
+ if (entryBytes > MAX_ENTRY_UNCOMPRESSED) {
130
+ fail(new Error(`attachment entry expands beyond the single-entry budget: ${name} exceeds ${MAX_ENTRY_UNCOMPRESSED} uncompressed bytes`))
131
+ stop()
132
+ return
133
+ }
134
+ if (self.decompressedBytes > MAX_TOTAL_UNCOMPRESSED) {
135
+ fail(new Error(`attachment archive expands beyond the unzip budget: more than ${MAX_TOTAL_UNCOMPRESSED} uncompressed bytes`))
136
+ stop()
137
+ return
138
+ }
139
+ chunks.push(chunk)
140
+ })
141
+ stream.on('end', () => {
142
+ if (settled) return
143
+ settled = true
144
+ resolve()
145
+ })
146
+ stream.on('error', (error) => fail(error))
147
+ })
148
+ return Buffer.concat(chunks, entryBytes)
149
+ }
150
+ }
151
+
152
+ /** Reject an archive before any entry is decompressed. */
38
153
  async function loadZip(buffer) {
39
154
  const zip = await JSZip.loadAsync(buffer)
40
- let total = 0
41
- for (const file of Object.values(zip.files)) {
42
- const size = file?._data?.uncompressedSize ?? 0
43
- total += size
44
- if (total > MAX_UNZIP_BYTES) throw new Error('attachment expands beyond 20 MB')
155
+ const names = Object.keys(zip.files)
156
+ if (names.length > MAX_ZIP_ENTRIES) {
157
+ throw new Error(`attachment archive has too many ZIP entries: ${names.length} (limit ${MAX_ZIP_ENTRIES})`)
158
+ }
159
+ let declaredTotal = 0
160
+ for (const name of names) {
161
+ const entry = zip.files[name]
162
+ if (!entry || entry.dir) continue
163
+ const declared = declaredUncompressedSize(entry) ?? 0
164
+ if (declared > MAX_ENTRY_UNCOMPRESSED) {
165
+ throw new Error(`attachment entry too large: ${name} declares ${declared} uncompressed bytes (limit ${MAX_ENTRY_UNCOMPRESSED})`)
166
+ }
167
+ declaredTotal += declared
168
+ if (declaredTotal > MAX_TOTAL_UNCOMPRESSED) {
169
+ throw new Error(`attachment archive expands beyond the unzip budget: declares ${declaredTotal} uncompressed bytes (limit ${MAX_TOTAL_UNCOMPRESSED})`)
170
+ }
45
171
  }
46
- return zip
172
+ return new ZipBudgetReader(zip)
47
173
  }
48
174
 
49
175
  function docxText(xml) {
@@ -55,11 +181,11 @@ function docxText(xml) {
55
181
  .replace(/\n{3,}/g, '\n\n').trim()
56
182
  }
57
183
 
58
- async function pptxText(zip) {
59
- const slides = Object.keys(zip.files).filter((n) => /^ppt\/slides\/slide\d+\.xml$/.test(n)).sort((a, b) => a.localeCompare(b, undefined, { numeric: true }))
184
+ async function pptxText(reader) {
185
+ const slides = reader.names().filter((n) => /^ppt\/slides\/slide\d+\.xml$/.test(n)).sort((a, b) => a.localeCompare(b, undefined, { numeric: true }))
60
186
  const out = []
61
187
  for (const name of slides) {
62
- const xml = await zip.file(name)?.async('string')
188
+ const xml = await reader.readText(name)
63
189
  if (!xml) continue
64
190
  const texts = [...xml.matchAll(/<a:t>([\s\S]*?)<\/a:t>/g)].map((m) => decodeXml(m[1]))
65
191
  if (texts.length) out.push(`# ${name.split('/').pop()}\n${texts.join('\n')}`)
@@ -68,11 +194,11 @@ async function pptxText(zip) {
68
194
  }
69
195
 
70
196
  export async function ofdText(buffer) {
71
- const zip = await loadZip(buffer)
72
- const names = Object.keys(zip.files).filter((n) => /(?:^|\/)Content\.xml$/i.test(n) || /(?:^|\/)Pages\/[^/]+\.xml$/i.test(n)).sort()
197
+ const reader = await loadZip(buffer)
198
+ const names = reader.names().filter((n) => /(?:^|\/)Content\.xml$/i.test(n) || /(?:^|\/)Pages\/[^/]+\.xml$/i.test(n)).sort()
73
199
  const out = []
74
200
  for (const name of names) {
75
- const xml = await zip.file(name)?.async('string')
201
+ const xml = await reader.readText(name)
76
202
  if (!xml) continue
77
203
  const texts = [...xml.matchAll(/<(?:ofd:)?TextCode[^>]*>([\s\S]*?)<\/(?:ofd:)?TextCode>/gi)].map((m) => decodeXml(m[1]))
78
204
  if (texts.length) out.push(texts.join('\n'))
@@ -99,20 +225,20 @@ async function pdfText(path) {
99
225
  }
100
226
  }
101
227
 
102
- async function xlsxText(zip) {
103
- const shared = await zip.file('xl/sharedStrings.xml')?.async('string')
228
+ async function xlsxText(reader) {
104
229
  const out = []
230
+ const shared = await reader.readText('xl/sharedStrings.xml')
105
231
  if (shared) out.push(...[...shared.matchAll(/<t[^>]*>([\s\S]*?)<\/t>/g)].map((m) => decodeXml(m[1])))
106
232
  if (!out.length) {
107
- for (const name of Object.keys(zip.files).filter((n) => /^xl\/worksheets\/sheet\d+\.xml$/.test(n))) {
108
- const xml = await zip.file(name)?.async('string')
233
+ for (const name of reader.names().filter((n) => /^xl\/worksheets\/sheet\d+\.xml$/.test(n))) {
234
+ const xml = await reader.readText(name)
109
235
  if (xml) out.push(...[...xml.matchAll(/<t[^>]*>([\s\S]*?)<\/t>/g)].map((m) => decodeXml(m[1])))
110
236
  }
111
237
  }
112
238
  return out.filter(Boolean).join('\n').trim()
113
239
  }
114
240
 
115
- export async function extractAttachment(input, { cwd = process.cwd() } = {}) {
241
+ export async function extractAttachment(input, { cwd = process.cwd(), hmacKey } = {}) {
116
242
  const path = safePath(cwd, input)
117
243
  const info = await stat(path)
118
244
  if (!info.isFile()) throw new Error(`not a file: ${input}`)
@@ -122,36 +248,36 @@ export async function extractAttachment(input, { cwd = process.cwd() } = {}) {
122
248
  if (TEXT_EXT.has(ext)) {
123
249
  const raw = await readFile(path, 'utf8')
124
250
  const { text, truncated } = truncate(raw)
125
- return { name, path, kind: ext.slice(1), text, truncated, bytes: info.size }
251
+ return withSafety({ name, path, kind: ext.slice(1), text, truncated, bytes: info.size }, { hmacKey })
126
252
  }
127
253
  if (OFFICE_EXT.has(ext)) {
128
- const zip = await loadZip(await readFile(path))
254
+ const reader = await loadZip(await readFile(path))
129
255
  const raw = ext === '.docx'
130
- ? docxText(await zip.file('word/document.xml')?.async('string') ?? '')
256
+ ? docxText(await reader.readText('word/document.xml') ?? '')
131
257
  : ext === '.pptx'
132
- ? await pptxText(zip)
133
- : await xlsxText(zip)
258
+ ? await pptxText(reader)
259
+ : await xlsxText(reader)
134
260
  const { text, truncated } = truncate(raw)
135
261
  if (!text) throw new Error(`no extractable text in ${name}`)
136
- return { name, path, kind: ext.slice(1), text, truncated, bytes: info.size }
262
+ return withSafety({ name, path, kind: ext.slice(1), text, truncated, bytes: info.size }, { hmacKey })
137
263
  }
138
264
  if (ext === '.ofd') {
139
265
  const raw = await ofdText(await readFile(path))
140
266
  const { text, truncated } = truncate(raw)
141
267
  if (!text) throw new Error(`no extractable text in ${name} (OFD image-only?)`)
142
- return { name, path, kind: 'ofd', text, truncated, bytes: info.size }
268
+ return withSafety({ name, path, kind: 'ofd', text, truncated, bytes: info.size }, { hmacKey })
143
269
  }
144
270
  if (IMAGE_EXT.has(ext)) {
145
271
  if (!ocrConfigured()) throw new Error(`image OCR is not configured; set ADVISERS_OCR_CMD to a local OCR executable (${name})`)
146
272
  const raw = await runLocalOcr(path, { cwd })
147
273
  const { text, truncated } = truncate(raw)
148
- return { name, path, kind: 'ocr', text, truncated, bytes: info.size }
274
+ return withSafety({ name, path, kind: 'ocr', text, truncated, bytes: info.size }, { hmacKey })
149
275
  }
150
276
  if (ext === '.pdf') {
151
277
  const raw = await pdfText(path)
152
278
  const { text, truncated } = truncate(raw)
153
279
  if (!text) throw new Error(`no extractable text in ${name} (scanned PDF?)`)
154
- return { name, path, kind: 'pdf', text, truncated, bytes: info.size }
280
+ return withSafety({ name, path, kind: 'pdf', text, truncated, bytes: info.size }, { hmacKey })
155
281
  }
156
282
  throw new Error(`unsupported attachment type: ${ext || '(none)'}`)
157
283
  }
package/lib/capture.js CHANGED
@@ -1,8 +1,63 @@
1
1
  import { chatCompletion } from './model.js'
2
2
  import { describeMessageTime, normalizeDue } from './relative-date.js'
3
+ import { classify, redact, safetyErrorFrom, buildAuditReason } from './safety/index.js'
3
4
 
4
5
  const MAX_CANDIDATES = 5
5
6
  const MAX_CHARS = 20_000
7
+ const CAPTURE_SURFACE = 'capture.input'
8
+
9
+ /**
10
+ * Shared capture safety gate (also reused by lib/mcp-server.js).
11
+ *
12
+ * Classifies text with the capture surface (local-file policy: non-CSAM blocks
13
+ * are downgraded to confirm; CSAM stays a hard block) and throws a typed,
14
+ * text-free ContentSafetyError on block. Callers must run this before any model
15
+ * call or candidate write so a blocked input never produces a candidate.
16
+ */
17
+ export function inspectCaptureText(text, { surface = CAPTURE_SURFACE, hmacKey } = {}) {
18
+ const result = classify(text, { surface, hmacKey })
19
+ if (result.action === 'block') throw safetyErrorFrom(result, { surface })
20
+ return result
21
+ }
22
+
23
+ function safetyWarningOf(result, direction) {
24
+ if (!result || result.action === 'allow') return null
25
+ const surface = result.surface || CAPTURE_SURFACE
26
+ const ruleIds = Array.isArray(result.ruleIds) ? [...result.ruleIds] : []
27
+ return {
28
+ direction,
29
+ action: result.action,
30
+ code: result.code,
31
+ categories: Array.isArray(result.categories) ? [...result.categories] : [],
32
+ ruleIds,
33
+ surface,
34
+ rulesetVersion: result.rulesetVersion,
35
+ evidenceHash: result.evidenceHash || null,
36
+ crisis: Boolean(result.crisis),
37
+ auditReason: buildAuditReason({
38
+ rulesetVersion: result.rulesetVersion,
39
+ surface,
40
+ action: result.action,
41
+ code: result.code,
42
+ ruleIds,
43
+ evidenceHash: result.evidenceHash,
44
+ }),
45
+ }
46
+ }
47
+
48
+ /**
49
+ * Redact PII/secrets from the candidate payload only. The quote stays verbatim:
50
+ * it is checked against the source text, so redacting it would break the
51
+ * verbatim contract. Redaction is one-way and never returns the original value.
52
+ */
53
+ function redactCandidatePayload(candidate) {
54
+ return {
55
+ ...candidate,
56
+ action: redact(candidate.action).text,
57
+ uncertainty: redact(candidate.uncertainty).text,
58
+ ...(candidate.creditor === undefined ? {} : { creditor: redact(candidate.creditor).text }),
59
+ }
60
+ }
6
61
 
7
62
  function normalize(text) {
8
63
  return String(text || '').replace(/\r\n/g, '\n').replace(/[ \t]+/g, ' ').trim()
@@ -70,12 +125,17 @@ export function validateCaptureCandidates(candidates, text, { now = new Date(),
70
125
  return out
71
126
  }
72
127
 
73
- export async function captureCandidates({ text, apiKey, baseUrl, model, fetchImpl = fetch, date, source, now = new Date() }) {
128
+ export async function captureCandidates({ text, apiKey, baseUrl, model, fetchImpl = fetch, date, source, now = new Date(), safetyKey }) {
74
129
  const input = normalize(text)
75
130
  if (!input) return { candidates: [], usage: undefined }
76
131
  if (input.length > MAX_CHARS) throw new Error(`capture input is too long (max ${MAX_CHARS} chars)`)
77
132
  const messageDate = date ? new Date(date) : now
78
133
  if (!Number.isFinite(messageDate.getTime())) throw new Error('capture message date is invalid')
134
+ // Fail closed before the model call: a blocked input produces no candidate.
135
+ const inputSafety = inspectCaptureText(input, { hmacKey: safetyKey })
136
+ const safetyWarnings = []
137
+ const inputWarning = safetyWarningOf(inputSafety, 'input')
138
+ if (inputWarning) safetyWarnings.push(inputWarning)
79
139
  const result = await chatCompletion({
80
140
  messages: [
81
141
  { role: 'system', content: '你是严谨的中文工作秘书。只输出 JSON,不输出解释。' },
@@ -83,6 +143,26 @@ export async function captureCandidates({ text, apiKey, baseUrl, model, fetchImp
83
143
  ],
84
144
  apiKey, baseUrl, model, fetchImpl,
85
145
  task: 'capture', thinking: true, effort: 'high', maxTokens: 4096,
146
+ safetyKey,
86
147
  })
87
- return { candidates: validateCaptureCandidates(parseCaptureResponse(result.content), input, { now, messageDate }), usage: result.usage, raw: result }
148
+ const validated = validateCaptureCandidates(parseCaptureResponse(result.content), input, { now, messageDate })
149
+ const candidates = []
150
+ for (const candidate of validated) {
151
+ // A model-invented candidate that is itself block-level must not become a
152
+ // pending row; it is dropped (the batch continues) and only a structural
153
+ // warning is reported - never the original text.
154
+ const candidateSafety = classify(`${candidate.action}\n${candidate.quote}`, { surface: CAPTURE_SURFACE, hmacKey: safetyKey })
155
+ if (candidateSafety.action === 'block') {
156
+ const dropped = safetyWarningOf(candidateSafety, 'output')
157
+ if (dropped) safetyWarnings.push({ ...dropped, dropped: true })
158
+ continue
159
+ }
160
+ candidates.push(redactCandidatePayload(candidate))
161
+ }
162
+ return {
163
+ candidates,
164
+ usage: result.usage,
165
+ raw: result,
166
+ ...(safetyWarnings.length ? { safetyWarnings } : {}),
167
+ }
88
168
  }