@try-works/dsh-memory 0.1.32 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/STATUS.md +29 -2
- package/package.json +1 -1
- package/src/assets/describe.js +181 -0
- package/src/assets/entry.js +169 -0
- package/src/assets/index.js +648 -0
- package/src/assets/lfs.js +169 -0
- package/src/assets/objects.js +199 -0
- package/src/assets/policy.js +259 -0
- package/src/config.js +12 -0
- package/src/defaults.js +11 -0
- package/src/entry.js +58 -13
- package/src/forget.js +7 -0
- package/src/host.js +202 -3
- package/src/refresh.js +7 -2
- package/src/search.js +47 -3
- package/src/tools.js +735 -10
|
@@ -0,0 +1,648 @@
|
|
|
1
|
+
// The asset store: the contract surface the agent tools use.
|
|
2
|
+
//
|
|
3
|
+
// An asset is an ENTRY (metadata + documentation, searchable) plus a BLOB (content-addressed bytes).
|
|
4
|
+
// This module orchestrates the pipeline from docs/asset-store-proposal.md section 5.5:
|
|
5
|
+
// classify -> toggle -> size -> quota -> sha256/dedupe -> LFS gate -> blob -> entry -> inventory
|
|
6
|
+
//
|
|
7
|
+
// It is a catalogue, not a text index: nothing is extracted from a file, and no query path here
|
|
8
|
+
// walks the blob tree. `list` and `usage` read entries only; `get` touches exactly one blob.
|
|
9
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'
|
|
10
|
+
import { basename, dirname, join, relative } from 'node:path'
|
|
11
|
+
|
|
12
|
+
import { allocateId, nowIso, parseEntries, scopeIdFor } from '../entry.js'
|
|
13
|
+
import { appendToTopic, layerIndexPath, layerText, readLayerTopics, rebuildLayerIndex } from '../layers.js'
|
|
14
|
+
import { updateIndex } from '../fts.js'
|
|
15
|
+
import { sanitizeKey } from '../workspace-key.js'
|
|
16
|
+
import { record } from '../telemetry.js'
|
|
17
|
+
import { DEFAULT_CONFIG } from '../defaults.js'
|
|
18
|
+
import {
|
|
19
|
+
ASSET_CLASSES,
|
|
20
|
+
addQuotaNotice as addQuotaNoticeAt,
|
|
21
|
+
canonicalMime,
|
|
22
|
+
classify,
|
|
23
|
+
extensionForMime,
|
|
24
|
+
normalizeMime,
|
|
25
|
+
quotaBytesFor,
|
|
26
|
+
readPolicy,
|
|
27
|
+
readQuotaNotices as readQuotaNoticesAt,
|
|
28
|
+
resolveToggles,
|
|
29
|
+
toggleRefusal,
|
|
30
|
+
writePolicy,
|
|
31
|
+
} from './policy.js'
|
|
32
|
+
import {
|
|
33
|
+
deleteObjectFiles,
|
|
34
|
+
objectBytesOnDisk,
|
|
35
|
+
readInventory,
|
|
36
|
+
readObject,
|
|
37
|
+
releaseObjectRef,
|
|
38
|
+
sha256,
|
|
39
|
+
upsertObjectRef,
|
|
40
|
+
writeObject,
|
|
41
|
+
} from './objects.js'
|
|
42
|
+
import { parseAssetEntry, renderAssetEntry } from './entry.js'
|
|
43
|
+
import { ensureLfs } from './lfs.js'
|
|
44
|
+
|
|
45
|
+
// The four policy-facing pieces are re-exported from here: tools import ONE module.
|
|
46
|
+
export { classify, readPolicy, resolveToggles, writePolicy } from './policy.js'
|
|
47
|
+
|
|
48
|
+
/** Standard error shape. A failure always names a stable code from the contract. */
|
|
49
|
+
function fail(code, message, detail) {
|
|
50
|
+
return detail === undefined ? { ok: false, code, message } : { ok: false, code, message, detail }
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** A store may be a MemoryStore or its root path; the tools hold one, the tests the other. */
|
|
54
|
+
function rootOf(store) {
|
|
55
|
+
if (typeof store === 'string' && store !== '') return store
|
|
56
|
+
if (store !== null && store !== undefined && typeof store.root === 'string' && store.root !== '') return store.root
|
|
57
|
+
throw new Error('assets: expected a MemoryStore or a store root path')
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Read the stored quota notices, newest first. An automatic capture cannot ask a question, so it
|
|
62
|
+
* records the wall it met here and `memory_status` reads it. Accepts a store or a root path.
|
|
63
|
+
*/
|
|
64
|
+
export function readQuotaNotices(store) {
|
|
65
|
+
return readQuotaNoticesAt(rootOf(store))
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Record (and dedupe by scope+key) a quota notice. Accepts a store or a root path. */
|
|
69
|
+
export function addQuotaNotice(store, notice = {}) {
|
|
70
|
+
return addQuotaNoticeAt(rootOf(store), notice)
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function numberOr(value, fallback) {
|
|
74
|
+
const number = Number(value)
|
|
75
|
+
return Number.isFinite(number) ? number : fallback
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function nowValue(now) {
|
|
79
|
+
if (typeof now === 'function') now = now()
|
|
80
|
+
if (now instanceof Date) return Number.isNaN(now.getTime()) ? nowIso() : now.toISOString()
|
|
81
|
+
if (typeof now === 'string' && now !== '') return now
|
|
82
|
+
if (typeof now === 'number' && Number.isFinite(now)) return new Date(now).toISOString()
|
|
83
|
+
return nowIso()
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function relPath(root, file) { return relative(root, file).replace(/\\/g, '/') }
|
|
87
|
+
|
|
88
|
+
/** The workspace directories present in the store. */
|
|
89
|
+
function workspaceKeys(root) {
|
|
90
|
+
let entries = []
|
|
91
|
+
try { entries = readdirSync(join(root, 'workspaces'), { withFileTypes: true }) } catch { return [] }
|
|
92
|
+
return entries.filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort()
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Selectors for a scope argument: global, one workspace, or both (the default). `null` means the
|
|
97
|
+
* caller asked for a workspace without a usable key.
|
|
98
|
+
*/
|
|
99
|
+
function selectorsFor(root, scope, key) {
|
|
100
|
+
if (scope === 'global') return [{ scope: 'global', key: undefined }]
|
|
101
|
+
if (scope === 'workspace') {
|
|
102
|
+
const safe = sanitizeKey(key)
|
|
103
|
+
return safe === undefined ? null : [{ scope: 'workspace', key: safe }]
|
|
104
|
+
}
|
|
105
|
+
return [{ scope: 'global', key: undefined }, ...workspaceKeys(root).map((workspaceKey) => ({ scope: 'workspace', key: workspaceKey }))]
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** Asset entries of one scope. Reads entry files only - never an object. */
|
|
109
|
+
function assetsInScope(root, scope, key) {
|
|
110
|
+
let topics = []
|
|
111
|
+
try { topics = readLayerTopics(root, scope, key, 'asset') } catch { return [] }
|
|
112
|
+
const assets = []
|
|
113
|
+
for (const topic of topics) {
|
|
114
|
+
const file = relPath(root, topic.file)
|
|
115
|
+
for (const entry of topic.entries) {
|
|
116
|
+
const asset = parseAssetEntry(entry)
|
|
117
|
+
if (asset === undefined) continue
|
|
118
|
+
assets.push({ ...asset, scope, key, file })
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
return assets
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function collectAssets(root, selectors) {
|
|
125
|
+
const assets = []
|
|
126
|
+
for (const selector of selectors ?? []) assets.push(...assetsInScope(root, selector.scope, selector.key))
|
|
127
|
+
return assets
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function isLive(asset) { return asset.status !== 'superseded' && asset.status !== 'deleted' }
|
|
131
|
+
|
|
132
|
+
/** The list projection, exactly the fields the contract names. */
|
|
133
|
+
function publicAsset(asset) {
|
|
134
|
+
return {
|
|
135
|
+
id: asset.id,
|
|
136
|
+
name: asset.name,
|
|
137
|
+
class: asset.class,
|
|
138
|
+
mime: asset.mime,
|
|
139
|
+
bytes: asset.bytes,
|
|
140
|
+
sha256: asset.sha256,
|
|
141
|
+
date: asset.date,
|
|
142
|
+
heading: asset.heading,
|
|
143
|
+
file: asset.file,
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** The richer metadata `getAsset` returns; the tools may report it verbatim. */
|
|
148
|
+
function assetMetadata(asset) {
|
|
149
|
+
const metadata = {
|
|
150
|
+
...publicAsset(asset),
|
|
151
|
+
scope: asset.scope,
|
|
152
|
+
// `key` is undefined for a global asset. Optional fields are OMITTED, never blanked: a tool
|
|
153
|
+
// result carrying `undefined` is not lossless JSON (the harness rejects it, and
|
|
154
|
+
// JSON.stringify silently drops the key, so the model would see a different value than the
|
|
155
|
+
// tool returned). The sweep at the end applies that rule to every field below at once.
|
|
156
|
+
...(asset.key === undefined ? {} : { key: asset.key }),
|
|
157
|
+
status: asset.status,
|
|
158
|
+
origin: asset.origin,
|
|
159
|
+
usedFor: asset.usedFor,
|
|
160
|
+
description: asset.description,
|
|
161
|
+
whatItIs: asset.whatItIs,
|
|
162
|
+
whenToRead: asset.whenToRead,
|
|
163
|
+
}
|
|
164
|
+
for (const field of Object.keys(metadata)) if (metadata[field] === undefined) delete metadata[field]
|
|
165
|
+
return metadata
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Per-scope byte maps built from the entries and the inventory. The inventory holds BYTES; only the
|
|
170
|
+
* entries hold SCOPE, so the two are joined here once and reused by usage and the quota gate.
|
|
171
|
+
*/
|
|
172
|
+
function collectUsage(root) {
|
|
173
|
+
const assets = collectAssets(root, selectorsFor(root, 'all')).filter(isLive)
|
|
174
|
+
const inventory = readInventory(root)
|
|
175
|
+
const scopes = new Map([['global', new Map()]])
|
|
176
|
+
for (const asset of assets) {
|
|
177
|
+
if (!asset.sha256) continue
|
|
178
|
+
const scopeKey = asset.scope === 'global' ? 'global' : asset.key
|
|
179
|
+
let map = scopes.get(scopeKey)
|
|
180
|
+
if (map === undefined) { map = new Map(); scopes.set(scopeKey, map) }
|
|
181
|
+
if (!map.has(asset.sha256)) map.set(asset.sha256, asset.bytes)
|
|
182
|
+
}
|
|
183
|
+
return { assets, inventory, scopes }
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function bytesOfObject(inventory, hash, fallback) {
|
|
187
|
+
const stored = inventory[hash]
|
|
188
|
+
const bytes = stored === undefined || stored === null ? NaN : Number(stored.bytes)
|
|
189
|
+
return Number.isFinite(bytes) && bytes >= 0 ? bytes : numberOr(fallback, 0)
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function sumScope(inventory, map) {
|
|
193
|
+
if (map === undefined) return 0
|
|
194
|
+
let total = 0
|
|
195
|
+
for (const [hash, fallback] of map) total += bytesOfObject(inventory, hash, fallback)
|
|
196
|
+
return total
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function largestInScope(root, { scope, key }) {
|
|
200
|
+
const selectors = selectorsFor(root, scope, key)
|
|
201
|
+
if (selectors === null) return []
|
|
202
|
+
return collectAssets(root, selectors)
|
|
203
|
+
.filter(isLive)
|
|
204
|
+
.sort((a, b) => b.bytes - a.bytes)
|
|
205
|
+
.slice(0, 3)
|
|
206
|
+
.map((asset) => ({ name: asset.name, bytes: asset.bytes, id: asset.id }))
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Per-scope byte usage + quotas, read from entries and the inventory only (no blob walk).
|
|
211
|
+
*
|
|
212
|
+
* - `global` / `workspaces`: logical bytes stored in each scope. An object deduped into two scopes
|
|
213
|
+
* counts in both, because deleting one scope's entry does not free the bytes for the other.
|
|
214
|
+
* - `totalBytes`: the unique object bytes the store actually holds, for the GitHub LFS ceiling.
|
|
215
|
+
* - `perClass`: how many live assets each class holds.
|
|
216
|
+
* - `quota`: the effective quota per scope (the policy override, else the configured value).
|
|
217
|
+
*/
|
|
218
|
+
export function usage(store, cfg = {}) {
|
|
219
|
+
const root = rootOf(store)
|
|
220
|
+
const policy = readPolicy(root)
|
|
221
|
+
const { assets, inventory, scopes } = collectUsage(root)
|
|
222
|
+
const global = sumScope(inventory, scopes.get('global'))
|
|
223
|
+
const workspaces = {}
|
|
224
|
+
for (const [scopeKey, map] of scopes) {
|
|
225
|
+
if (scopeKey === 'global') continue
|
|
226
|
+
workspaces[scopeKey] = sumScope(inventory, map)
|
|
227
|
+
}
|
|
228
|
+
const quotaWorkspaces = {}
|
|
229
|
+
const quotaKeys = new Set([...Object.keys(workspaces), ...Object.keys(policy.workspaces ?? {})])
|
|
230
|
+
for (const workspaceKey of quotaKeys) {
|
|
231
|
+
quotaWorkspaces[workspaceKey] = quotaBytesFor(cfg, policy, { scope: 'workspace', key: workspaceKey })
|
|
232
|
+
}
|
|
233
|
+
const uniqueHashes = new Map()
|
|
234
|
+
for (const asset of assets) {
|
|
235
|
+
if (!asset.sha256 || uniqueHashes.has(asset.sha256)) continue
|
|
236
|
+
uniqueHashes.set(asset.sha256, asset.bytes)
|
|
237
|
+
}
|
|
238
|
+
let totalBytes = 0
|
|
239
|
+
for (const [hash, fallback] of uniqueHashes) totalBytes += bytesOfObject(inventory, hash, fallback)
|
|
240
|
+
const perClass = { documents: 0, media: 0, data: 0 }
|
|
241
|
+
for (const asset of assets) {
|
|
242
|
+
if (Object.hasOwn(perClass, asset.class)) perClass[asset.class] += 1
|
|
243
|
+
}
|
|
244
|
+
return {
|
|
245
|
+
global,
|
|
246
|
+
workspaces,
|
|
247
|
+
quota: {
|
|
248
|
+
global: quotaBytesFor(cfg, policy, { scope: 'global' }),
|
|
249
|
+
workspaces: quotaWorkspaces,
|
|
250
|
+
},
|
|
251
|
+
totalBytes,
|
|
252
|
+
perClass,
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* The numbers `memory_status` needs, without the per-scope detail. A convenience over `usage`.
|
|
258
|
+
*/
|
|
259
|
+
export function assetBytesReport(store, cfg = {}) {
|
|
260
|
+
const report = usage(store, cfg)
|
|
261
|
+
return {
|
|
262
|
+
totalBytes: report.totalBytes,
|
|
263
|
+
perClass: report.perClass,
|
|
264
|
+
global: report.global,
|
|
265
|
+
workspaces: report.workspaces,
|
|
266
|
+
quota: report.quota,
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// --- capture --------------------------------------------------------------------------------
|
|
271
|
+
|
|
272
|
+
function toBytes(value) {
|
|
273
|
+
if (Buffer.isBuffer(value)) return value
|
|
274
|
+
if (value instanceof Uint8Array) return Buffer.from(value)
|
|
275
|
+
if (value instanceof ArrayBuffer) return Buffer.from(new Uint8Array(value))
|
|
276
|
+
if (typeof value === 'string') return Buffer.from(value, 'utf8')
|
|
277
|
+
return undefined
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
function loadBytes(input) {
|
|
281
|
+
if (input.bytes !== undefined && input.bytes !== null) {
|
|
282
|
+
const bytes = toBytes(input.bytes)
|
|
283
|
+
if (bytes === undefined) return fail('missing-bytes', 'bytes must be a Buffer, Uint8Array, ArrayBuffer or string')
|
|
284
|
+
return { ok: true, bytes }
|
|
285
|
+
}
|
|
286
|
+
if (input.path !== undefined && input.path !== null && String(input.path) !== '') {
|
|
287
|
+
const file = String(input.path)
|
|
288
|
+
if (!existsSync(file)) return fail('missing-bytes', 'no file at ' + file)
|
|
289
|
+
try { return { ok: true, bytes: readFileSync(file) } } catch (error) {
|
|
290
|
+
return fail('read-failed', 'could not read ' + file + ': ' + String(error?.message ?? error))
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
return fail('missing-bytes', 'putAsset needs bytes or a path')
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
function firstLine(text, max = 200) {
|
|
297
|
+
const line = String(text ?? '').split('\n').map((part) => part.trim()).find((part) => part.length > 0)
|
|
298
|
+
if (line === undefined) return undefined
|
|
299
|
+
const collapsed = line.replace(/\s+/g, ' ')
|
|
300
|
+
if (collapsed.length <= max) return collapsed
|
|
301
|
+
return [...collapsed].slice(0, max - 1).join('') + '…'
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
function entryHeading(name) {
|
|
305
|
+
const cleaned = String(name ?? '').replace(/[\r\n]+/g, ' ').trim()
|
|
306
|
+
if (cleaned === '') return 'asset'
|
|
307
|
+
const points = [...cleaned]
|
|
308
|
+
return points.length > 80 ? points.slice(0, 79).join('') + '…' : cleaned
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
function deterministicUsedFor(context, name) {
|
|
312
|
+
const message = firstLine(context?.userMessage, 160)
|
|
313
|
+
if (message !== undefined) return 'Captured while handling: ' + message
|
|
314
|
+
const task = firstLine(context?.task, 120)
|
|
315
|
+
if (task !== undefined) return 'Captured while working on: ' + task
|
|
316
|
+
return 'Stored for later reuse: ' + name
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
/**
|
|
320
|
+
* Deterministic documentation from capture context. This is the floor the proposal promises:
|
|
321
|
+
* capture never fails for want of a description, and the two headings always exist so the entry is
|
|
322
|
+
* searchable by what it was captured for.
|
|
323
|
+
*/
|
|
324
|
+
function fallbackDocumentation(context, meta) {
|
|
325
|
+
const message = firstLine(context?.userMessage, 180)
|
|
326
|
+
const task = firstLine(context?.task, 140)
|
|
327
|
+
const tool = firstLine(context?.toolCall, 120)
|
|
328
|
+
const whatItIs = 'A ' + meta.class + ' file named "' + meta.name + '" (' + meta.mime + ', '
|
|
329
|
+
+ meta.bytes + ' bytes).' + (task === undefined ? '' : ' Captured while working on: ' + task + '.')
|
|
330
|
+
let whenToRead
|
|
331
|
+
if (message !== undefined) {
|
|
332
|
+
whenToRead = 'Read this when the user revisits the request it was captured for: "' + message
|
|
333
|
+
+ '". Open it before answering from memory alone.'
|
|
334
|
+
} else if (tool !== undefined) {
|
|
335
|
+
whenToRead = 'Read this when following up on "' + tool + '".'
|
|
336
|
+
} else {
|
|
337
|
+
whenToRead = 'Read this when the user needs the ' + meta.class + ' file "' + meta.name + '".'
|
|
338
|
+
}
|
|
339
|
+
return { whatItIs, whenToRead, notes: undefined }
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
function cleanText(value) {
|
|
343
|
+
const text = String(value ?? '').trim()
|
|
344
|
+
return text === '' ? undefined : text
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
function normalizeDocumentation(value) {
|
|
348
|
+
if (value === null || value === undefined || typeof value !== 'object') return undefined
|
|
349
|
+
const whatItIs = cleanText(value.whatItIs)
|
|
350
|
+
const whenToRead = cleanText(value.whenToRead)
|
|
351
|
+
if (whatItIs === undefined && whenToRead === undefined) return undefined
|
|
352
|
+
return {
|
|
353
|
+
whatItIs,
|
|
354
|
+
whenToRead,
|
|
355
|
+
notes: cleanText(value.notes),
|
|
356
|
+
description: cleanText(value.description),
|
|
357
|
+
usedFor: cleanText(value.usedFor),
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
/**
|
|
362
|
+
* Resolve the documentation. Order: explicit `documentation` (or whatItIs/whenToRead), then an
|
|
363
|
+
* injected `describe` function (await inside putAsset, one entry write), then the deterministic
|
|
364
|
+
* floor. Any missing section is filled from the floor so both headings always exist.
|
|
365
|
+
*/
|
|
366
|
+
async function resolveDocumentation(input, meta) {
|
|
367
|
+
const base = fallbackDocumentation(input.context, meta)
|
|
368
|
+
const supplied = normalizeDocumentation(input.documentation)
|
|
369
|
+
?? normalizeDocumentation({ whatItIs: input.whatItIs, whenToRead: input.whenToRead, notes: input.notes })
|
|
370
|
+
if (supplied !== undefined) {
|
|
371
|
+
return {
|
|
372
|
+
whatItIs: supplied.whatItIs ?? base.whatItIs,
|
|
373
|
+
whenToRead: supplied.whenToRead ?? base.whenToRead,
|
|
374
|
+
notes: supplied.notes,
|
|
375
|
+
description: supplied.description,
|
|
376
|
+
usedFor: supplied.usedFor,
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
if (typeof input.describe === 'function') {
|
|
380
|
+
try {
|
|
381
|
+
const described = normalizeDocumentation(await input.describe({
|
|
382
|
+
class: meta.class,
|
|
383
|
+
mime: meta.mime,
|
|
384
|
+
name: meta.name,
|
|
385
|
+
bytes: meta.bytes,
|
|
386
|
+
context: input.context,
|
|
387
|
+
llm: input.llm,
|
|
388
|
+
}))
|
|
389
|
+
if (described !== undefined) {
|
|
390
|
+
return {
|
|
391
|
+
whatItIs: described.whatItIs ?? base.whatItIs,
|
|
392
|
+
whenToRead: described.whenToRead ?? base.whenToRead,
|
|
393
|
+
notes: described.notes,
|
|
394
|
+
description: described.description,
|
|
395
|
+
usedFor: described.usedFor,
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
} catch { /* the deterministic floor is the contract; a model is an enhancement */ }
|
|
399
|
+
}
|
|
400
|
+
return base
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/**
|
|
404
|
+
* Store one asset: blob + entry + inventory, after toggles, size and quota checks.
|
|
405
|
+
* @see docs/asset-api-contract.md for the input/output shape and the stable error codes.
|
|
406
|
+
*/
|
|
407
|
+
export async function putAsset(store, cfg = {}, input = {}) {
|
|
408
|
+
const root = rootOf(store)
|
|
409
|
+
const scope = input.scope === 'global' ? 'global' : 'workspace'
|
|
410
|
+
const key = scope === 'workspace' ? sanitizeKey(input.key) : undefined
|
|
411
|
+
if (scope === 'workspace' && key === undefined) return fail('write-failed', 'workspace asset capture needs a workspace key')
|
|
412
|
+
|
|
413
|
+
const sourcePath = input.path === undefined || input.path === null || String(input.path) === '' ? undefined : String(input.path)
|
|
414
|
+
const suppliedName = input.name === undefined || input.name === null ? undefined : String(input.name)
|
|
415
|
+
const name = suppliedName ?? (sourcePath === undefined ? undefined : basename(sourcePath))
|
|
416
|
+
|
|
417
|
+
const claimed = input.class === undefined
|
|
418
|
+
? classify({ name, mime: input.mime })
|
|
419
|
+
: (ASSET_CLASSES.includes(String(input.class)) ? String(input.class) : undefined)
|
|
420
|
+
if (claimed === undefined) {
|
|
421
|
+
if (input.class !== undefined) {
|
|
422
|
+
return fail('unsupported-class', 'unsupported asset class "' + String(input.class) + '"; use documents, media or data')
|
|
423
|
+
}
|
|
424
|
+
return fail(
|
|
425
|
+
'unsupported-class',
|
|
426
|
+
'no asset class claims ' + JSON.stringify(name ?? normalizeMime(input.mime) ?? 'the input')
|
|
427
|
+
+ '; accepted: PDF/DOCX/PPTX/TXT/MD (documents), PNG/JPEG/WebP/GIF (media), CSV/TSV/JSON/XLSX (data)',
|
|
428
|
+
)
|
|
429
|
+
}
|
|
430
|
+
const assetClass = claimed
|
|
431
|
+
|
|
432
|
+
const refusal = toggleRefusal(cfg, root, { scope, key, class: assetClass })
|
|
433
|
+
if (refusal !== undefined) {
|
|
434
|
+
return fail('toggle-off', refusal.message, { scope, key: key ?? null, class: assetClass, level: refusal.level })
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
const loaded = loadBytes(input)
|
|
438
|
+
if (loaded.ok !== true) return loaded
|
|
439
|
+
const bytes = loaded.bytes
|
|
440
|
+
|
|
441
|
+
const maxBytes = numberOr(cfg?.assetsMaxBytes, DEFAULT_CONFIG.assetsMaxBytes)
|
|
442
|
+
if (bytes.length > maxBytes) {
|
|
443
|
+
return fail('file-too-large', 'asset is ' + bytes.length + ' bytes; the per-file limit is ' + maxBytes + ' bytes', { bytes: bytes.length, limit: maxBytes })
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
const mime = canonicalMime({ name, mime: input.mime })
|
|
447
|
+
const displayName = name !== undefined && String(name).trim() !== ''
|
|
448
|
+
? String(name).trim()
|
|
449
|
+
: 'asset.' + extensionForMime(mime)
|
|
450
|
+
const hash = sha256(bytes)
|
|
451
|
+
|
|
452
|
+
// Quota: refuse, surface the figures, never prune. The caller asks the user; this function only
|
|
453
|
+
// reports. `allowOverQuota` is the one-shot "forgive this capture" bypass.
|
|
454
|
+
const { inventory, scopes } = collectUsage(root)
|
|
455
|
+
const scopeKey = scope === 'global' ? 'global' : key
|
|
456
|
+
const usedBytes = sumScope(inventory, scopes.get(scopeKey))
|
|
457
|
+
const dedupedInScope = scopes.get(scopeKey)?.has(hash) === true
|
|
458
|
+
const quotaBytes = quotaBytesFor(cfg, readPolicy(root), { scope, key })
|
|
459
|
+
const projected = usedBytes + (dedupedInScope ? 0 : bytes.length)
|
|
460
|
+
if (input.allowOverQuota !== true && projected > quotaBytes) {
|
|
461
|
+
return fail(
|
|
462
|
+
'quota-exceeded',
|
|
463
|
+
'asset quota reached for ' + (scope === 'global' ? 'the global scope' : 'workspace ' + key)
|
|
464
|
+
+ ' (' + usedBytes + ' of ' + quotaBytes + ' bytes used)',
|
|
465
|
+
{ scope, key: key ?? null, usedBytes, quotaBytes, largest: largestInScope(root, { scope, key }) },
|
|
466
|
+
)
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
// Nothing is written until the LFS gate passes, so a machine without Git LFS cannot accumulate
|
|
470
|
+
// objects that would be committed as raw bytes instead of LFS-tracked content.
|
|
471
|
+
const lfs = await ensureLfs(root, { run: input.runner })
|
|
472
|
+
if (lfs.ok !== true) return fail(lfs.code ?? 'lfs-unavailable', lfs.message)
|
|
473
|
+
|
|
474
|
+
const deduped = inventory[hash] !== undefined
|
|
475
|
+
const object = writeObject(root, mime, hash, bytes)
|
|
476
|
+
if (object.ok !== true) return object
|
|
477
|
+
|
|
478
|
+
const documentation = await resolveDocumentation(input, { class: assetClass, mime, name: displayName, bytes: bytes.length })
|
|
479
|
+
const usedFor = cleanText(input.usedFor) ?? documentation.usedFor ?? deterministicUsedFor(input.context, displayName)
|
|
480
|
+
|
|
481
|
+
let id
|
|
482
|
+
let file
|
|
483
|
+
try {
|
|
484
|
+
const scopeId = scopeIdFor(scope, key)
|
|
485
|
+
id = allocateId(layerText(root, scope, key, 'asset'), scopeId, 'asset')
|
|
486
|
+
const heading = entryHeading(displayName)
|
|
487
|
+
const block = renderAssetEntry({
|
|
488
|
+
id,
|
|
489
|
+
source: input.source !== undefined && input.source !== null ? String(input.source) : 'agent',
|
|
490
|
+
date: nowValue(input.now),
|
|
491
|
+
importance: input.importance ?? 2,
|
|
492
|
+
status: 'active',
|
|
493
|
+
heading,
|
|
494
|
+
asset: {
|
|
495
|
+
class: assetClass,
|
|
496
|
+
mime,
|
|
497
|
+
bytes: bytes.length,
|
|
498
|
+
sha256: hash,
|
|
499
|
+
name: displayName,
|
|
500
|
+
origin: input.origin ?? sourcePath ?? 'inline',
|
|
501
|
+
usedFor,
|
|
502
|
+
description: documentation.description,
|
|
503
|
+
},
|
|
504
|
+
documentation,
|
|
505
|
+
})
|
|
506
|
+
file = appendToTopic(root, scope, key, 'asset', heading, block)
|
|
507
|
+
rebuildLayerIndex(root, scope, key, 'asset')
|
|
508
|
+
updateIndex(root, [file, layerIndexPath(root, scope, key, 'asset')])
|
|
509
|
+
} catch (error) {
|
|
510
|
+
return fail('write-failed', 'could not write the asset entry: ' + String(error?.message ?? error))
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
upsertObjectRef(root, { sha256: hash, bytes: bytes.length, mime, created: nowValue(input.now) })
|
|
514
|
+
record(root, 'asset-added', { detail: { class: assetClass, bytes: bytes.length, sha256: hash, dedupe: deduped } })
|
|
515
|
+
return {
|
|
516
|
+
ok: true,
|
|
517
|
+
entryId: id,
|
|
518
|
+
sha256: hash,
|
|
519
|
+
bytes: bytes.length,
|
|
520
|
+
deduped,
|
|
521
|
+
entryPath: relPath(root, file),
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
// --- retrieval ------------------------------------------------------------------------------
|
|
526
|
+
|
|
527
|
+
/**
|
|
528
|
+
* Resolve one asset to metadata and optionally copy the bytes out. The only path that reads a blob,
|
|
529
|
+
* and it reads exactly one. A missing object or an LFS pointer is reported as `missing-bytes` with
|
|
530
|
+
* the fix, never fabricated.
|
|
531
|
+
*/
|
|
532
|
+
export async function getAsset(store, cfg = {}, { id, sha256: wantedSha, toPath } = {}) {
|
|
533
|
+
const root = rootOf(store)
|
|
534
|
+
const wantedId = id === undefined || id === null ? undefined : String(id)
|
|
535
|
+
const wantedHash = wantedSha === undefined || wantedSha === null ? undefined : String(wantedSha)
|
|
536
|
+
if (wantedId === undefined && wantedHash === undefined) return fail('not-found', 'getAsset needs an asset id or sha256')
|
|
537
|
+
|
|
538
|
+
const candidates = collectAssets(root, selectorsFor(root, 'all'))
|
|
539
|
+
let hit
|
|
540
|
+
if (wantedId !== undefined) hit = candidates.find((asset) => asset.id === wantedId)
|
|
541
|
+
if (hit === undefined && wantedHash !== undefined) hit = candidates.find((asset) => asset.sha256 === wantedHash && isLive(asset))
|
|
542
|
+
if (hit === undefined) {
|
|
543
|
+
return fail('not-found', 'no asset ' + (wantedId !== undefined ? 'entry with id ' + wantedId : 'with sha256 ' + wantedHash))
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
const read = readObject(root, hit.mime, hit.sha256)
|
|
547
|
+
if (read.ok !== true) return fail(read.code, read.message)
|
|
548
|
+
const metadata = assetMetadata(hit)
|
|
549
|
+
|
|
550
|
+
if (toPath === undefined || toPath === null || String(toPath) === '') {
|
|
551
|
+
return { ok: true, metadata, bytes: read.bytes }
|
|
552
|
+
}
|
|
553
|
+
let target = String(toPath)
|
|
554
|
+
if (target.endsWith('/') || target.endsWith('\\')) target = join(target, hit.name ?? 'asset')
|
|
555
|
+
else try { if (statSync(target).isDirectory()) target = join(target, hit.name ?? 'asset') } catch { /* path does not exist yet: a file */ }
|
|
556
|
+
try {
|
|
557
|
+
mkdirSync(dirname(target), { recursive: true })
|
|
558
|
+
writeFileSync(target, read.bytes)
|
|
559
|
+
} catch (error) {
|
|
560
|
+
return fail('write-failed', 'could not copy the asset to ' + target + ': ' + String(error?.message ?? error))
|
|
561
|
+
}
|
|
562
|
+
return { ok: true, metadata, path: target }
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
/**
|
|
566
|
+
* List asset entries in a scope, newest first, bounded. Reads entries only; an object on disk that
|
|
567
|
+
* is missing, a pointer, or unreadable does not affect the listing.
|
|
568
|
+
*/
|
|
569
|
+
export function listAssets(store, { scope, key, class: assetClass, limit } = {}) {
|
|
570
|
+
const root = rootOf(store)
|
|
571
|
+
const selectors = selectorsFor(root, scope === undefined || scope === null ? 'all' : String(scope), key)
|
|
572
|
+
if (selectors === null) return fail('not-found', 'workspace scope needs a valid workspace key')
|
|
573
|
+
let assets = collectAssets(root, selectors).filter(isLive)
|
|
574
|
+
if (assetClass !== undefined && assetClass !== null && String(assetClass) !== '') {
|
|
575
|
+
assets = assets.filter((asset) => asset.class === String(assetClass))
|
|
576
|
+
}
|
|
577
|
+
assets.sort((a, b) => {
|
|
578
|
+
const byDate = String(b.date ?? '').localeCompare(String(a.date ?? ''))
|
|
579
|
+
return byDate !== 0 ? byDate : String(b.id ?? '').localeCompare(String(a.id ?? ''))
|
|
580
|
+
})
|
|
581
|
+
const bound = limit === undefined || limit === null
|
|
582
|
+
? 50
|
|
583
|
+
: Math.max(1, Math.min(1000, Math.floor(numberOr(limit, 50))))
|
|
584
|
+
return { ok: true, assets: assets.slice(0, bound).map(publicAsset) }
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
/**
|
|
588
|
+
* Remove one entry block from its topic file. The entry is deleted (not superseded): an asset entry
|
|
589
|
+
* without its blob is a promise the store cannot keep, and the contract says forget deletes.
|
|
590
|
+
*/
|
|
591
|
+
function removeEntryBlock(file, id) {
|
|
592
|
+
let text
|
|
593
|
+
try { text = readFileSync(file, 'utf8') } catch { return fail('read-failed', 'could not read ' + file) }
|
|
594
|
+
const target = parseEntries(text).find((entry) => entry.id === id)
|
|
595
|
+
if (target === undefined) return fail('not-found', 'entry ' + id + ' is not in ' + file)
|
|
596
|
+
const lines = text.split('\n')
|
|
597
|
+
const before = lines.slice(0, target.startLine - 1)
|
|
598
|
+
const after = lines.slice(target.endLine)
|
|
599
|
+
while (after.length > 0 && after[0].trim() === '') after.shift()
|
|
600
|
+
const next = [...before, ...(after.length > 0 ? ['', ...after] : [])].join('\n')
|
|
601
|
+
try { writeFileSync(file, next, 'utf8') } catch (error) {
|
|
602
|
+
return fail('write-failed', 'could not rewrite ' + file + ': ' + String(error?.message ?? error))
|
|
603
|
+
}
|
|
604
|
+
return { ok: true }
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
/**
|
|
608
|
+
* Delete the entry; decrement the object's refcount and delete the blob when it reaches zero. The
|
|
609
|
+
* logical bytes freed are reported so a caller can explain what changed.
|
|
610
|
+
*/
|
|
611
|
+
export function forgetAsset(store, cfg = {}, { id } = {}) {
|
|
612
|
+
const root = rootOf(store)
|
|
613
|
+
const wantedId = id === undefined || id === null ? undefined : String(id)
|
|
614
|
+
if (wantedId === undefined) return fail('not-found', 'forgetAsset needs an asset id')
|
|
615
|
+
const hit = collectAssets(root, selectorsFor(root, 'all')).find((asset) => asset.id === wantedId)
|
|
616
|
+
if (hit === undefined) return fail('not-found', 'no asset entry with id ' + wantedId)
|
|
617
|
+
|
|
618
|
+
const file = join(root, ...String(hit.file).split('/'))
|
|
619
|
+
const removed = removeEntryBlock(file, wantedId)
|
|
620
|
+
if (removed.ok !== true) return removed
|
|
621
|
+
try {
|
|
622
|
+
rebuildLayerIndex(root, hit.scope, hit.key, 'asset')
|
|
623
|
+
updateIndex(root, [file, layerIndexPath(root, hit.scope, hit.key, 'asset')])
|
|
624
|
+
} catch (error) {
|
|
625
|
+
return fail('write-failed', 'could not update the asset index: ' + String(error?.message ?? error))
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
const release = releaseObjectRef(root, hit.sha256)
|
|
629
|
+
let objectDeleted = false
|
|
630
|
+
let bytesFreed = 0
|
|
631
|
+
if (release.found) {
|
|
632
|
+
if (release.refs === 0) {
|
|
633
|
+
const deleted = deleteObjectFiles(root, hit.sha256)
|
|
634
|
+
objectDeleted = deleted > 0
|
|
635
|
+
bytesFreed = deleted > 0 ? release.bytes : 0
|
|
636
|
+
}
|
|
637
|
+
} else {
|
|
638
|
+
// Inventory drift: fall back to the remaining entries that still reference the object. If none
|
|
639
|
+
// do, the blob is unreachable and is cleaned up like a normal last-reference delete.
|
|
640
|
+
const remaining = collectAssets(root, selectorsFor(root, 'all')).filter((asset) => asset.sha256 === hit.sha256)
|
|
641
|
+
if (remaining.length === 0) {
|
|
642
|
+
const deleted = deleteObjectFiles(root, hit.sha256)
|
|
643
|
+
objectDeleted = deleted > 0
|
|
644
|
+
bytesFreed = deleted > 0 ? objectBytesOnDisk(root, hit.sha256) : 0
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
return { ok: true, entryId: wantedId, objectDeleted, bytesFreed }
|
|
648
|
+
}
|