@try-works/dsh-memory 0.1.32 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/STATUS.md +29 -2
- package/package.json +1 -1
- package/src/assets/describe.js +181 -0
- package/src/assets/entry.js +169 -0
- package/src/assets/index.js +648 -0
- package/src/assets/lfs.js +169 -0
- package/src/assets/objects.js +199 -0
- package/src/assets/policy.js +259 -0
- package/src/config.js +12 -0
- package/src/defaults.js +11 -0
- package/src/entry.js +58 -13
- package/src/forget.js +7 -0
- package/src/host.js +202 -3
- package/src/refresh.js +7 -2
- package/src/search.js +47 -3
- package/src/tools.js +735 -10
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
// Git LFS: the transport for asset objects.
|
|
2
|
+
//
|
|
3
|
+
// The command runner is injectable and shaped like GitRunner.run in src/git.js: `(argv, opts)`
|
|
4
|
+
// where argv EXCLUDES the `git` executable. Production passes the host's subprocess-backed
|
|
5
|
+
// runner; tests pass a fake, so no test ever runs git. Nothing here executes at import time.
|
|
6
|
+
import { execFile } from 'node:child_process'
|
|
7
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
|
|
8
|
+
import { dirname, join } from 'node:path'
|
|
9
|
+
|
|
10
|
+
/** The rule that makes every object travel through LFS rather than the normal git plumbing. */
|
|
11
|
+
export const GITATTRIBUTES_LINE = 'assets/objects/** filter=lfs diff=lfs merge=lfs -text'
|
|
12
|
+
|
|
13
|
+
/** The first line of every LFS pointer file (spec v1). */
|
|
14
|
+
const LFS_POINTER_PREFIX = 'version https://git-lfs.github.com/spec/v1'
|
|
15
|
+
|
|
16
|
+
/** The git LFS binary probe result is cached per store for the life of the process. */
|
|
17
|
+
const availableCache = new Map()
|
|
18
|
+
|
|
19
|
+
/** In-flight `git lfs install --local` per store; a second capture awaits the first. */
|
|
20
|
+
const installPromises = new Map()
|
|
21
|
+
|
|
22
|
+
export function gitattributesPath(root) { return join(root, '.gitattributes') }
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Default runner: node:child_process with the same result shape as GitRunner.run.
|
|
26
|
+
* Used only when no runner is injected (the plugin passes the host's subprocess runner).
|
|
27
|
+
*/
|
|
28
|
+
export function defaultCommandRunner(argv, opts = {}) {
|
|
29
|
+
return new Promise((resolve) => {
|
|
30
|
+
execFile('git', argv, {
|
|
31
|
+
cwd: opts.cwd,
|
|
32
|
+
timeout: opts.timeout ?? 15_000,
|
|
33
|
+
windowsHide: true,
|
|
34
|
+
maxBuffer: 4 * 1024 * 1024,
|
|
35
|
+
}, (error, stdout, stderr) => {
|
|
36
|
+
if (!error) {
|
|
37
|
+
resolve({ exitCode: 0, stdout: String(stdout ?? ''), stderr: String(stderr ?? '') })
|
|
38
|
+
return
|
|
39
|
+
}
|
|
40
|
+
const code = typeof error.code === 'number' ? error.code : (error.code === 'ENOENT' ? 127 : 1)
|
|
41
|
+
resolve({
|
|
42
|
+
exitCode: code,
|
|
43
|
+
stdout: String(stdout ?? ''),
|
|
44
|
+
stderr: String(stderr ?? '') || String(error.message ?? error),
|
|
45
|
+
})
|
|
46
|
+
})
|
|
47
|
+
})
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* True when the content is a Git LFS POINTER rather than the bytes it stands for. A pointer begins
|
|
52
|
+
* with the spec version line; the version line alone is not enough if it is not followed by the
|
|
53
|
+
* line break the format requires, so a PDF whose first words happen to match is not misread.
|
|
54
|
+
*/
|
|
55
|
+
export function isLfsPointer(content) {
|
|
56
|
+
if (content === undefined || content === null) return false
|
|
57
|
+
const text = Buffer.isBuffer(content) ? content.subarray(0, 200).toString('utf8') : String(content).slice(0, 200)
|
|
58
|
+
if (!text.startsWith(LFS_POINTER_PREFIX)) return false
|
|
59
|
+
const next = text[LFS_POINTER_PREFIX.length]
|
|
60
|
+
return next === undefined || next === '\n' || next === '\r'
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Add the object filter rule to `.gitattributes` if it is not already there. Idempotent: the
|
|
65
|
+
* existing bytes are never rewritten, only appended to.
|
|
66
|
+
* @returns {{changed: boolean, created: boolean, error: Error|null}}
|
|
67
|
+
*/
|
|
68
|
+
export function ensureGitattributes(root) {
|
|
69
|
+
const file = gitattributesPath(root)
|
|
70
|
+
let existing = ''
|
|
71
|
+
try { existing = existsSync(file) ? readFileSync(file, 'utf8') : '' } catch (error) {
|
|
72
|
+
return { changed: false, created: false, error }
|
|
73
|
+
}
|
|
74
|
+
if (existing.split('\n').some((line) => line.trim() === GITATTRIBUTES_LINE)) {
|
|
75
|
+
return { changed: false, created: false, error: null }
|
|
76
|
+
}
|
|
77
|
+
const separator = existing.length > 0 && !existing.endsWith('\n') ? '\n' : ''
|
|
78
|
+
try {
|
|
79
|
+
mkdirSync(dirname(file), { recursive: true })
|
|
80
|
+
writeFileSync(file, existing + separator + GITATTRIBUTES_LINE + '\n', 'utf8')
|
|
81
|
+
} catch (error) {
|
|
82
|
+
return { changed: false, created: false, error }
|
|
83
|
+
}
|
|
84
|
+
return { changed: true, created: existing.length === 0, error: null }
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Is `git lfs` available to this process? Probes `git lfs version` once per store and caches the
|
|
89
|
+
* success; a failure is NOT cached, so installing LFS later works without a restart.
|
|
90
|
+
*/
|
|
91
|
+
export async function lfsAvailable({ root, run = defaultCommandRunner, refresh = false } = {}) {
|
|
92
|
+
const cacheKey = String(root ?? '')
|
|
93
|
+
if (!refresh && availableCache.get(cacheKey) === true) return true
|
|
94
|
+
let result
|
|
95
|
+
try { result = await run(['lfs', 'version'], { cwd: root }) } catch { result = undefined }
|
|
96
|
+
const ok = result !== undefined && result !== null && (result.exitCode ?? result.code) === 0
|
|
97
|
+
if (ok) availableCache.set(cacheKey, true)
|
|
98
|
+
return ok
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Run `git lfs install --local` once per store. Only meaningful inside a git work tree: outside
|
|
103
|
+
* one there is nothing to configure, so that is reported as a no-op rather than a failure.
|
|
104
|
+
* @returns {Promise<{ok: boolean, installed: boolean, reason?: string}>}
|
|
105
|
+
*/
|
|
106
|
+
export async function ensureLfsInstalled(root, { run = defaultCommandRunner } = {}) {
|
|
107
|
+
if (!root || !existsSync(join(root, '.git'))) {
|
|
108
|
+
return { ok: true, installed: false, reason: 'store is not a git repository yet' }
|
|
109
|
+
}
|
|
110
|
+
if (!installPromises.has(root)) {
|
|
111
|
+
installPromises.set(root, (async () => {
|
|
112
|
+
let result
|
|
113
|
+
try { result = await run(['lfs', 'install', '--local'], { cwd: root }) } catch (error) {
|
|
114
|
+
return { ok: false, installed: false, reason: String(error?.message ?? error) }
|
|
115
|
+
}
|
|
116
|
+
if (result === undefined || result === null || (result.exitCode ?? result.code) !== 0) {
|
|
117
|
+
const reason = String(result?.stderr || result?.stdout || 'git lfs install --local failed').trim()
|
|
118
|
+
return { ok: false, installed: false, reason }
|
|
119
|
+
}
|
|
120
|
+
return { ok: true, installed: true }
|
|
121
|
+
})())
|
|
122
|
+
}
|
|
123
|
+
return installPromises.get(root)
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* The capture-time gate: the filter rule is in place, the binary exists, and (inside a repo) LFS is
|
|
128
|
+
* enabled locally. Objects are written only after this passes, so a store can never silently fill
|
|
129
|
+
* with pointer files that look like bytes.
|
|
130
|
+
* @returns {Promise<{ok: boolean, installed: boolean, gitattributesChanged: boolean, code?: string, message?: string}>}
|
|
131
|
+
*/
|
|
132
|
+
export async function ensureLfs(root, options = {}) {
|
|
133
|
+
const run = options.run ?? defaultCommandRunner
|
|
134
|
+
const attributes = ensureGitattributes(root)
|
|
135
|
+
if (attributes.error) {
|
|
136
|
+
return {
|
|
137
|
+
ok: false,
|
|
138
|
+
installed: false,
|
|
139
|
+
gitattributesChanged: false,
|
|
140
|
+
code: 'write-failed',
|
|
141
|
+
message: 'could not write .gitattributes: ' + String(attributes.error?.message ?? attributes.error),
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
const available = await lfsAvailable({ root, run, refresh: options.refresh })
|
|
145
|
+
if (!available) {
|
|
146
|
+
return {
|
|
147
|
+
ok: false,
|
|
148
|
+
installed: false,
|
|
149
|
+
gitattributesChanged: attributes.changed,
|
|
150
|
+
code: 'lfs-unavailable',
|
|
151
|
+
message: 'git lfs is not available on this machine; install Git LFS before storing assets',
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
const installed = await ensureLfsInstalled(root, { run })
|
|
155
|
+
if (!installed.ok) {
|
|
156
|
+
return {
|
|
157
|
+
ok: false,
|
|
158
|
+
installed: false,
|
|
159
|
+
gitattributesChanged: attributes.changed,
|
|
160
|
+
code: 'lfs-unavailable',
|
|
161
|
+
message: 'git lfs is available but could not be enabled in this repository: ' + (installed.reason ?? 'unknown error'),
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
return {
|
|
165
|
+
ok: true,
|
|
166
|
+
installed: installed.installed,
|
|
167
|
+
gitattributesChanged: attributes.changed,
|
|
168
|
+
}
|
|
169
|
+
}
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
// Content-addressed object blobs + the inventory inside state/assets.json.
|
|
2
|
+
//
|
|
3
|
+
// One object per unique content, independent of name or scope: `assets/objects/<ab>/<sha256>.<ext>`.
|
|
4
|
+
// `refs` counts the asset entries across all scopes that point at the object, so `forget` can delete
|
|
5
|
+
// the blob at zero without walking the blob tree. Everything here is synchronous file work; the
|
|
6
|
+
// only caller that may touch bytes is `get` (and the write itself).
|
|
7
|
+
import { createHash } from 'node:crypto'
|
|
8
|
+
import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
|
9
|
+
import { dirname, join } from 'node:path'
|
|
10
|
+
import { canonicalMime, extensionForMime } from './policy.js'
|
|
11
|
+
import { isLfsPointer } from './lfs.js'
|
|
12
|
+
|
|
13
|
+
const SHA256 = /^[a-f0-9]{64}$/
|
|
14
|
+
|
|
15
|
+
/** sha256 of content, lowercase hex. Accepts a Buffer or string. */
|
|
16
|
+
export function sha256(content) {
|
|
17
|
+
return createHash('sha256').update(content).digest('hex')
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function objectsRoot(root) { return join(root, 'assets', 'objects') }
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Root-relative object path: assets/objects/<sha256[0:2]>/<sha256>.<ext>.
|
|
24
|
+
* The extension is derived from the MIME, not the file name, so the same content always resolves to
|
|
25
|
+
* the same path regardless of what it was called when it was captured.
|
|
26
|
+
*/
|
|
27
|
+
export function objectRelPath(mime, hash) {
|
|
28
|
+
const digest = String(hash ?? '')
|
|
29
|
+
if (!SHA256.test(digest)) throw new Error('invalid sha256: ' + digest)
|
|
30
|
+
return 'assets/objects/' + digest.slice(0, 2) + '/' + digest + '.' + extensionForMime(mime)
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function objectPath(root, mime, hash) {
|
|
34
|
+
return join(root, ...objectRelPath(mime, hash).split('/'))
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Every existing file for one digest, whatever its extension. Used for read and delete fallback. */
|
|
38
|
+
export function objectCandidates(root, hash) {
|
|
39
|
+
const digest = String(hash ?? '')
|
|
40
|
+
if (!SHA256.test(digest)) return []
|
|
41
|
+
const dir = join(objectsRoot(root), digest.slice(0, 2))
|
|
42
|
+
let names = []
|
|
43
|
+
try { names = readdirSync(dir).filter((name) => name.startsWith(digest + '.')) } catch { return [] }
|
|
44
|
+
return names.map((name) => join(dir, name))
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Read one object's bytes.
|
|
49
|
+
* @returns {{ok: true, bytes: Buffer, file: string} | {ok: false, code: string, message: string}}
|
|
50
|
+
* A missing file and an LFS pointer are both `missing-bytes`: a 130-byte pointer is not the PDF the
|
|
51
|
+
* caller asked for, and pretending otherwise is the failure this store exists to avoid.
|
|
52
|
+
*/
|
|
53
|
+
export function readObject(root, mime, hash) {
|
|
54
|
+
const primary = objectPath(root, mime, hash)
|
|
55
|
+
const candidates = existsSync(primary)
|
|
56
|
+
? [primary, ...objectCandidates(root, hash).filter((file) => file !== primary)]
|
|
57
|
+
: objectCandidates(root, hash)
|
|
58
|
+
if (candidates.length === 0) {
|
|
59
|
+
return {
|
|
60
|
+
ok: false,
|
|
61
|
+
code: 'missing-bytes',
|
|
62
|
+
message: 'the bytes for asset ' + hash + ' are missing from assets/objects; run `git lfs pull` in the memory store',
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
let lastError
|
|
66
|
+
for (const file of candidates) {
|
|
67
|
+
let bytes
|
|
68
|
+
try { bytes = readFileSync(file) } catch (error) { lastError = error; continue }
|
|
69
|
+
if (isLfsPointer(bytes)) {
|
|
70
|
+
return {
|
|
71
|
+
ok: false,
|
|
72
|
+
code: 'missing-bytes',
|
|
73
|
+
message: 'asset ' + hash + ' is a Git LFS pointer, not the bytes; run `git lfs pull` in the memory store',
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return { ok: true, bytes, file }
|
|
77
|
+
}
|
|
78
|
+
return {
|
|
79
|
+
ok: false,
|
|
80
|
+
code: 'read-failed',
|
|
81
|
+
message: 'could not read asset ' + hash + ': ' + String(lastError?.message ?? lastError),
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Write one object, creating the sharded directory. An existing complete file is left alone; a
|
|
87
|
+
* pointer or a truncated file is replaced with the real bytes, so re-capturing heals a clone that
|
|
88
|
+
* was made without LFS.
|
|
89
|
+
* @returns {{ok: true, rel: string, file: string, written: boolean} | {ok: false, code: string, message: string}}
|
|
90
|
+
*/
|
|
91
|
+
export function writeObject(root, mime, hash, bytes) {
|
|
92
|
+
const rel = objectRelPath(mime, hash)
|
|
93
|
+
const file = join(root, ...rel.split('/'))
|
|
94
|
+
if (existsSync(file)) {
|
|
95
|
+
try {
|
|
96
|
+
const existing = readFileSync(file)
|
|
97
|
+
if (!isLfsPointer(existing) && existing.length === bytes.length) return { ok: true, rel, file, written: false }
|
|
98
|
+
} catch { /* rewrite below is the safer answer */ }
|
|
99
|
+
}
|
|
100
|
+
try {
|
|
101
|
+
mkdirSync(dirname(file), { recursive: true })
|
|
102
|
+
writeFileSync(file, bytes)
|
|
103
|
+
return { ok: true, rel, file, written: true }
|
|
104
|
+
} catch (error) {
|
|
105
|
+
return { ok: false, code: 'write-failed', message: 'could not write asset object: ' + String(error?.message ?? error) }
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Delete every file for a digest. @returns how many files were removed. */
|
|
110
|
+
export function deleteObjectFiles(root, hash) {
|
|
111
|
+
let deleted = 0
|
|
112
|
+
for (const file of objectCandidates(root, hash)) {
|
|
113
|
+
try { rmSync(file, { force: true }); deleted += 1 } catch { /* a stuck file must not hide the refcount result */ }
|
|
114
|
+
}
|
|
115
|
+
return deleted
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** Bytes physically present for a digest (used when the inventory has drifted). */
|
|
119
|
+
export function objectBytesOnDisk(root, hash) {
|
|
120
|
+
let total = 0
|
|
121
|
+
for (const file of objectCandidates(root, hash)) {
|
|
122
|
+
try { total += statSync(file).size } catch { /* unreadable file contributes nothing */ }
|
|
123
|
+
}
|
|
124
|
+
return total
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// --- inventory ------------------------------------------------------------------------------
|
|
128
|
+
// The inventory is one map inside state/assets.json. It is read and written through the policy
|
|
129
|
+
// module's state helpers so a policy write and an inventory write share one merge path.
|
|
130
|
+
|
|
131
|
+
import { readAssetState, writeAssetState } from './policy.js'
|
|
132
|
+
|
|
133
|
+
function numberOr(value, fallback) {
|
|
134
|
+
const number = Number(value)
|
|
135
|
+
return Number.isFinite(number) && number >= 0 ? number : fallback
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** The object inventory: sha256 -> { bytes, mime, refs, created }. */
|
|
139
|
+
export function readInventory(root) {
|
|
140
|
+
return readAssetState(root).objects
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** The inventory record for one digest, or undefined. */
|
|
144
|
+
export function inventoryObject(root, hash) {
|
|
145
|
+
const record = readInventory(root)[hash]
|
|
146
|
+
return record === undefined ? undefined : record
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Record one entry's reference to an object. Creates the inventory record on first use and
|
|
151
|
+
* increments `refs` on every later capture, including a dedupe inside the same scope.
|
|
152
|
+
* @returns {{added: boolean, refs: number, object: object}}
|
|
153
|
+
*/
|
|
154
|
+
export function upsertObjectRef(root, { sha256: hash, bytes, mime, created } = {}) {
|
|
155
|
+
const state = readAssetState(root)
|
|
156
|
+
const existing = state.objects[hash]
|
|
157
|
+
if (existing !== undefined && existing !== null && typeof existing === 'object') {
|
|
158
|
+
existing.refs = numberOr(existing.refs, 0) + 1
|
|
159
|
+
if (!numberOr(existing.bytes, 0) && numberOr(bytes, 0)) existing.bytes = numberOr(bytes, 0)
|
|
160
|
+
if (!existing.mime && mime) existing.mime = mime
|
|
161
|
+
writeAssetState(root, state)
|
|
162
|
+
return { added: false, refs: existing.refs, object: existing }
|
|
163
|
+
}
|
|
164
|
+
const record = {
|
|
165
|
+
bytes: numberOr(bytes, 0),
|
|
166
|
+
mime: String(mime ?? 'application/octet-stream'),
|
|
167
|
+
refs: 1,
|
|
168
|
+
created: created ?? new Date().toISOString(),
|
|
169
|
+
}
|
|
170
|
+
state.objects[hash] = record
|
|
171
|
+
writeAssetState(root, state)
|
|
172
|
+
return { added: true, refs: 1, object: record }
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Drop one reference. At zero the record is removed from the inventory and its logical bytes are
|
|
177
|
+
* reported so the caller can delete the file and say what was freed.
|
|
178
|
+
* @returns {{found: boolean, refs: number, bytes: number, removed: boolean}}
|
|
179
|
+
*/
|
|
180
|
+
export function releaseObjectRef(root, hash) {
|
|
181
|
+
const state = readAssetState(root)
|
|
182
|
+
const existing = state.objects[hash]
|
|
183
|
+
if (existing === undefined || existing === null || typeof existing !== 'object') {
|
|
184
|
+
return { found: false, refs: 0, bytes: 0, removed: false }
|
|
185
|
+
}
|
|
186
|
+
const refs = Math.max(0, numberOr(existing.refs, 0) - 1)
|
|
187
|
+
if (refs === 0) {
|
|
188
|
+
const bytes = numberOr(existing.bytes, 0)
|
|
189
|
+
delete state.objects[hash]
|
|
190
|
+
writeAssetState(root, state)
|
|
191
|
+
return { found: true, refs: 0, bytes, removed: true }
|
|
192
|
+
}
|
|
193
|
+
existing.refs = refs
|
|
194
|
+
writeAssetState(root, state)
|
|
195
|
+
return { found: true, refs, bytes: 0, removed: false }
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// Re-exported so callers that only need the canonical mime/path pair do not import two modules.
|
|
199
|
+
export { canonicalMime }
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
// Asset policy + classification (pure, except the state/assets.json read/write).
|
|
2
|
+
//
|
|
3
|
+
// `state/assets.json` holds two different things in one file: the per-scope policy (toggles and
|
|
4
|
+
// quotas, which a user or agent narrows) and the object inventory (what the store holds). Reading
|
|
5
|
+
// projects the policy; writing merges it back so a policy edit can never drop the inventory, and an
|
|
6
|
+
// inventory update can never drop an unknown policy field.
|
|
7
|
+
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
|
|
8
|
+
import { join } from 'node:path'
|
|
9
|
+
import { DEFAULT_CONFIG } from '../defaults.js'
|
|
10
|
+
import { sanitizeKey } from '../workspace-key.js'
|
|
11
|
+
|
|
12
|
+
/** The three asset classes, in the order they are reported. */
|
|
13
|
+
export const ASSET_CLASSES = ['documents', 'media', 'data']
|
|
14
|
+
|
|
15
|
+
/** Config field controlling each class. */
|
|
16
|
+
export const CLASS_TOGGLE = { documents: 'assetsDocuments', media: 'assetsMedia', data: 'assetsData' }
|
|
17
|
+
|
|
18
|
+
// One table: [mime, extension, class]. Everything else is derived from it, so the accepted set
|
|
19
|
+
// cannot drift between the classifier, the object path and the refusal message.
|
|
20
|
+
const TYPES = [
|
|
21
|
+
['application/pdf', 'pdf', 'documents'],
|
|
22
|
+
['application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'docx', 'documents'],
|
|
23
|
+
['application/vnd.openxmlformats-officedocument.presentationml.presentation', 'pptx', 'documents'],
|
|
24
|
+
['text/plain', 'txt', 'documents'],
|
|
25
|
+
['text/markdown', 'md', 'documents'],
|
|
26
|
+
['text/csv', 'csv', 'data'],
|
|
27
|
+
['text/tab-separated-values', 'tsv', 'data'],
|
|
28
|
+
['application/json', 'json', 'data'],
|
|
29
|
+
['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'xlsx', 'data'],
|
|
30
|
+
['image/png', 'png', 'media'],
|
|
31
|
+
['image/jpeg', 'jpg', 'media'],
|
|
32
|
+
['image/webp', 'webp', 'media'],
|
|
33
|
+
['image/gif', 'gif', 'media'],
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
const MIME_CLASS = new Map(TYPES.map(([mime, , assetClass]) => [mime, assetClass]))
|
|
37
|
+
const MIME_EXTENSION = new Map(TYPES.map(([mime, extension]) => [mime, extension]))
|
|
38
|
+
const EXTENSION_MIME = new Map()
|
|
39
|
+
for (const [mime, extension] of MIME_EXTENSION) EXTENSION_MIME.set(extension, mime)
|
|
40
|
+
// Names people actually have on disk, mapped onto the canonical mime above.
|
|
41
|
+
EXTENSION_MIME.set('jpeg', 'image/jpeg')
|
|
42
|
+
EXTENSION_MIME.set('markdown', 'text/markdown')
|
|
43
|
+
|
|
44
|
+
/** Lowercase mime without parameters. Undefined when absent. */
|
|
45
|
+
export function normalizeMime(mime) {
|
|
46
|
+
const text = String(mime ?? '').trim().toLowerCase()
|
|
47
|
+
if (text === '') return undefined
|
|
48
|
+
return text.split(';')[0].trim() || undefined
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Lowercase extension (no dot) from a file name, or undefined. */
|
|
52
|
+
export function extensionOfName(name) {
|
|
53
|
+
const base = String(name ?? '').split(/[\\/]/).pop() ?? ''
|
|
54
|
+
const at = base.lastIndexOf('.')
|
|
55
|
+
if (at <= 0 || at >= base.length - 1) return undefined
|
|
56
|
+
const extension = base.slice(at + 1).toLowerCase()
|
|
57
|
+
return /^[a-z0-9]+$/.test(extension) ? extension : undefined
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Canonical mime for a file name, when the extension is one we know. */
|
|
61
|
+
export function mimeForName(name) {
|
|
62
|
+
const extension = extensionOfName(name)
|
|
63
|
+
return extension === undefined ? undefined : EXTENSION_MIME.get(extension)
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Canonical extension used in an object path. Unknown mime -> 'bin'. */
|
|
67
|
+
export function extensionForMime(mime) {
|
|
68
|
+
const type = normalizeMime(mime)
|
|
69
|
+
return (type === undefined ? undefined : MIME_EXTENSION.get(type)) ?? 'bin'
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* The class that claims a file, from its mime or its name. An explicit, recognized mime wins over
|
|
74
|
+
* the name; an explicit mime nothing accepts wins too - a `.pdf` that is really a zip is not a
|
|
75
|
+
* document we can store, and silently believing the extension would be the worse answer.
|
|
76
|
+
* @returns {'documents'|'media'|'data'|undefined}
|
|
77
|
+
*/
|
|
78
|
+
export function classify({ name, mime } = {}) {
|
|
79
|
+
const explicit = normalizeMime(mime)
|
|
80
|
+
if (explicit !== undefined && explicit !== 'application/octet-stream') return MIME_CLASS.get(explicit)
|
|
81
|
+
const byName = mimeForName(name)
|
|
82
|
+
return byName === undefined ? undefined : MIME_CLASS.get(byName)
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** The mime stored on the object: an explicit recognized one, else the name's, else octet-stream. */
|
|
86
|
+
export function canonicalMime({ name, mime } = {}) {
|
|
87
|
+
const explicit = normalizeMime(mime)
|
|
88
|
+
if (explicit !== undefined && explicit !== 'application/octet-stream') return explicit
|
|
89
|
+
return mimeForName(name) ?? 'application/octet-stream'
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// --- policy file ----------------------------------------------------------------------------
|
|
93
|
+
|
|
94
|
+
export function policyPath(root) { return join(root, 'state', 'assets.json') }
|
|
95
|
+
|
|
96
|
+
function plainObject(value) {
|
|
97
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value) ? value : {}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function cfgBoolean(cfg, key, fallback) {
|
|
101
|
+
const value = cfg === undefined || cfg === null ? undefined : cfg[key]
|
|
102
|
+
return typeof value === 'boolean' ? value : fallback
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function cfgNumber(cfg, key, fallback) {
|
|
106
|
+
const value = cfg === undefined || cfg === null ? undefined : cfg[key]
|
|
107
|
+
return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : fallback
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* The whole state file, normalized. Exported for the inventory: one reader, so policy and objects
|
|
112
|
+
* can never disagree about what the file says.
|
|
113
|
+
*/
|
|
114
|
+
export function readAssetState(root) {
|
|
115
|
+
let parsed
|
|
116
|
+
try { parsed = JSON.parse(readFileSync(policyPath(root), 'utf8')) } catch { parsed = undefined }
|
|
117
|
+
const base = plainObject(parsed)
|
|
118
|
+
return {
|
|
119
|
+
...base,
|
|
120
|
+
version: Number.isInteger(base.version) ? base.version : 1,
|
|
121
|
+
global: plainObject(base.global),
|
|
122
|
+
workspaces: plainObject(base.workspaces),
|
|
123
|
+
objects: plainObject(base.objects),
|
|
124
|
+
quotaNotices: Array.isArray(base.quotaNotices) ? base.quotaNotices : [],
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export function writeAssetState(root, state) {
|
|
129
|
+
mkdirSync(join(root, 'state'), { recursive: true })
|
|
130
|
+
writeFileSync(policyPath(root), JSON.stringify(state, null, 2) + '\n', 'utf8')
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** The policy projection: toggles and quotas only, never the inventory. */
|
|
134
|
+
export function readPolicy(root) {
|
|
135
|
+
const state = readAssetState(root)
|
|
136
|
+
return { version: state.version, global: state.global, workspaces: state.workspaces }
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Merge a policy into the state file, preserving the inventory and unknown fields. A field absent
|
|
141
|
+
* from `policy` is left as it was, so a caller can write only the section it changed.
|
|
142
|
+
*/
|
|
143
|
+
export function writePolicy(root, policy = {}) {
|
|
144
|
+
const state = readAssetState(root)
|
|
145
|
+
const next = { ...state }
|
|
146
|
+
if (policy.version !== undefined) next.version = policy.version
|
|
147
|
+
if (policy.global !== undefined) next.global = plainObject(policy.global)
|
|
148
|
+
if (policy.workspaces !== undefined) next.workspaces = plainObject(policy.workspaces)
|
|
149
|
+
writeAssetState(root, next)
|
|
150
|
+
return next
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// --- toggle resolution ----------------------------------------------------------------------
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Effective toggles for one scope. Config is a hard upper bound; a missing policy field does not
|
|
157
|
+
* narrow anything. A class is on only when every level allows it, so a workspace can narrow itself
|
|
158
|
+
* but cannot re-enable a class the config or the global policy turned off.
|
|
159
|
+
* @returns {{enabled: boolean, documents: boolean, media: boolean, data: boolean}}
|
|
160
|
+
*/
|
|
161
|
+
export function resolveToggles(cfg, root, { scope, key } = {}) {
|
|
162
|
+
const enabled = cfgBoolean(cfg, 'assetsEnabled', DEFAULT_CONFIG.assetsEnabled)
|
|
163
|
+
const policy = readPolicy(root)
|
|
164
|
+
const workspaceKey = scope === 'workspace' ? sanitizeKey(key) : undefined
|
|
165
|
+
const workspacePolicy = workspaceKey === undefined ? {} : plainObject(policy.workspaces[workspaceKey])
|
|
166
|
+
const toggles = { enabled }
|
|
167
|
+
for (const assetClass of ASSET_CLASSES) {
|
|
168
|
+
const configOn = enabled && cfgBoolean(cfg, CLASS_TOGGLE[assetClass], DEFAULT_CONFIG[CLASS_TOGGLE[assetClass]])
|
|
169
|
+
const globalOn = policy.global[assetClass] !== false
|
|
170
|
+
const workspaceOn = scope === 'workspace' ? workspacePolicy[assetClass] !== false : true
|
|
171
|
+
toggles[assetClass] = configOn && globalOn && workspaceOn
|
|
172
|
+
}
|
|
173
|
+
return toggles
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Why a class may not be captured in this scope, or undefined when it may. Names the level so the
|
|
178
|
+
* refusal is actionable: "config", "workspace" or "global".
|
|
179
|
+
* @returns {{level: string, message: string}|undefined}
|
|
180
|
+
*/
|
|
181
|
+
export function toggleRefusal(cfg, root, { scope, key, class: assetClass } = {}) {
|
|
182
|
+
if (!cfgBoolean(cfg, 'assetsEnabled', DEFAULT_CONFIG.assetsEnabled)) {
|
|
183
|
+
return { level: 'config', message: assetClass + ' capture is disabled because the asset store is off in config (assetsEnabled)' }
|
|
184
|
+
}
|
|
185
|
+
if (!cfgBoolean(cfg, CLASS_TOGGLE[assetClass], DEFAULT_CONFIG[CLASS_TOGGLE[assetClass]])) {
|
|
186
|
+
return { level: 'config', message: assetClass + ' capture is disabled in config (' + CLASS_TOGGLE[assetClass] + ')' }
|
|
187
|
+
}
|
|
188
|
+
// Resolution order is config -> workspace -> global, and the refusal names the first level that
|
|
189
|
+
// narrowed, because that is the setting the reader needs to change.
|
|
190
|
+
const policy = readPolicy(root)
|
|
191
|
+
const workspaceKey = scope === 'workspace' ? sanitizeKey(key) : undefined
|
|
192
|
+
if (workspaceKey !== undefined) {
|
|
193
|
+
const workspacePolicy = plainObject(policy.workspaces[workspaceKey])
|
|
194
|
+
if (workspacePolicy[assetClass] === false) {
|
|
195
|
+
return { level: 'workspace', message: assetClass + ' capture is disabled by workspace policy for "' + workspaceKey + '"' }
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
if (policy.global[assetClass] === false) {
|
|
199
|
+
return { level: 'global', message: assetClass + ' capture is disabled by the global asset policy' }
|
|
200
|
+
}
|
|
201
|
+
return undefined
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/** Quota for one scope: the policy's stored override, else the configured value. */
|
|
205
|
+
export function quotaBytesFor(cfg, policy, { scope, key } = {}) {
|
|
206
|
+
if (scope === 'global') {
|
|
207
|
+
const stored = Number(policy?.global?.quotaBytes)
|
|
208
|
+
if (Number.isFinite(stored) && stored >= 0) return stored
|
|
209
|
+
return cfgNumber(cfg, 'assetsGlobalQuotaBytes', DEFAULT_CONFIG.assetsGlobalQuotaBytes)
|
|
210
|
+
}
|
|
211
|
+
const workspaceKey = sanitizeKey(key)
|
|
212
|
+
const stored = Number(workspaceKey === undefined ? undefined : policy?.workspaces?.[workspaceKey]?.quotaBytes)
|
|
213
|
+
if (Number.isFinite(stored) && stored >= 0) return stored
|
|
214
|
+
return cfgNumber(cfg, 'assetsWorkspaceQuotaBytes', DEFAULT_CONFIG.assetsWorkspaceQuotaBytes)
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// --- quota notices --------------------------------------------------------------------------
|
|
218
|
+
// An AUTOMATIC capture that hits the quota is not an interactive moment, so it records a notice
|
|
219
|
+
// instead of asking a question. One notice per scope+key: the newest figures win, because the
|
|
220
|
+
// reader wants the current wall, not its history.
|
|
221
|
+
|
|
222
|
+
function nonNegative(value) {
|
|
223
|
+
const number = Number(value)
|
|
224
|
+
return Number.isFinite(number) && number >= 0 ? number : 0
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
function normalizeNotice(notice = {}) {
|
|
228
|
+
const scope = notice.scope === 'global' ? 'global' : 'workspace'
|
|
229
|
+
return {
|
|
230
|
+
scope,
|
|
231
|
+
key: scope === 'global' ? null : (sanitizeKey(notice.key) ?? null),
|
|
232
|
+
at: String(notice.at ?? '') === '' ? new Date().toISOString() : String(notice.at),
|
|
233
|
+
usedBytes: nonNegative(notice.usedBytes),
|
|
234
|
+
quotaBytes: nonNegative(notice.quotaBytes),
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/** Record the quota wall an automatic capture met; replaces the notice for that scope+key. */
|
|
239
|
+
export function addQuotaNotice(root, notice = {}) {
|
|
240
|
+
const state = readAssetState(root)
|
|
241
|
+
const stored = normalizeNotice(notice)
|
|
242
|
+
const kept = state.quotaNotices.filter((item) => {
|
|
243
|
+
if (item === null || typeof item !== 'object') return false
|
|
244
|
+
const sameScope = (item.scope === 'global' ? 'global' : 'workspace') === stored.scope
|
|
245
|
+
const itemKey = item.scope === 'global' ? null : (sanitizeKey(item.key) ?? null)
|
|
246
|
+
return !(sameScope && itemKey === stored.key)
|
|
247
|
+
})
|
|
248
|
+
// Bounded: a notice collection is a status board, not a log.
|
|
249
|
+
writeAssetState(root, { ...state, quotaNotices: [...kept, stored].slice(-50) })
|
|
250
|
+
return stored
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/** Every stored quota notice, newest first. */
|
|
254
|
+
export function readQuotaNotices(root) {
|
|
255
|
+
return readAssetState(root).quotaNotices
|
|
256
|
+
.filter((notice) => notice !== null && typeof notice === 'object')
|
|
257
|
+
.map((notice) => normalizeNotice(notice))
|
|
258
|
+
.sort((a, b) => String(b.at).localeCompare(String(a.at)))
|
|
259
|
+
}
|
package/src/config.js
CHANGED
|
@@ -52,6 +52,18 @@ export const Config = z.object({
|
|
|
52
52
|
summaryMaxBullets: z.number().default(DEFAULT_CONFIG.summaryMaxBullets),
|
|
53
53
|
consolidateMaxPromptBytes: z.number().default(DEFAULT_CONFIG.consolidateMaxPromptBytes),
|
|
54
54
|
consolidateMaxDigestBytes: z.number().default(DEFAULT_CONFIG.consolidateMaxDigestBytes),
|
|
55
|
+
|
|
56
|
+
// --- config-file only: asset store ---
|
|
57
|
+
// A hard upper bound, on top of the per-scope policy in state/assets.json. Off by default:
|
|
58
|
+
// storing files is an explicit choice, and a disabled class refuses rather than drops.
|
|
59
|
+
assetsEnabled: z.boolean().default(DEFAULT_CONFIG.assetsEnabled),
|
|
60
|
+
assetsDocuments: z.boolean().default(DEFAULT_CONFIG.assetsDocuments),
|
|
61
|
+
assetsMedia: z.boolean().default(DEFAULT_CONFIG.assetsMedia),
|
|
62
|
+
assetsData: z.boolean().default(DEFAULT_CONFIG.assetsData),
|
|
63
|
+
assetsMaxBytes: z.number().default(DEFAULT_CONFIG.assetsMaxBytes),
|
|
64
|
+
assetsWorkspaceQuotaBytes: z.number().default(DEFAULT_CONFIG.assetsWorkspaceQuotaBytes),
|
|
65
|
+
assetsGlobalQuotaBytes: z.number().default(DEFAULT_CONFIG.assetsGlobalQuotaBytes),
|
|
66
|
+
assetsDescribe: z.boolean().default(DEFAULT_CONFIG.assetsDescribe),
|
|
55
67
|
})
|
|
56
68
|
|
|
57
69
|
export default Config
|
package/src/defaults.js
CHANGED
|
@@ -48,6 +48,17 @@ export const DEFAULT_CONFIG = Object.freeze({
|
|
|
48
48
|
rerankSearch: false,
|
|
49
49
|
injectionMaxChars: 3_600,
|
|
50
50
|
injectionRefreshMs: 900_000,
|
|
51
|
+
// Asset store (documents / media / data). The master switch is OFF: nothing is captured and
|
|
52
|
+
// every asset tool refuses until it is turned on. Once on, the three class toggles default to
|
|
53
|
+
// true so the common case is one switch; per-scope narrowing lives in state/assets.json.
|
|
54
|
+
assetsEnabled: false,
|
|
55
|
+
assetsDocuments: true,
|
|
56
|
+
assetsMedia: true,
|
|
57
|
+
assetsData: true,
|
|
58
|
+
assetsMaxBytes: 50 * 1024 * 1024, // 50 MB per file
|
|
59
|
+
assetsWorkspaceQuotaBytes: 1024 * 1024 * 1024, // 1 GB per workspace
|
|
60
|
+
assetsGlobalQuotaBytes: 1024 * 1024 * 1024, // 1 GB for the global scope
|
|
61
|
+
assetsDescribe: true, // one model call per captured asset
|
|
51
62
|
})
|
|
52
63
|
|
|
53
64
|
/**
|