circle-ir-ai 2.22.0 → 2.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +390 -0
- package/dist/agents/mastra/workflow.d.ts.map +1 -1
- package/dist/agents/mastra/workflow.js +41 -1
- package/dist/agents/mastra/workflow.js.map +1 -1
- package/dist/cache/backend.d.ts +52 -0
- package/dist/cache/backend.d.ts.map +1 -0
- package/dist/cache/backend.js +19 -0
- package/dist/cache/backend.js.map +1 -0
- package/dist/cache/backends/index.d.ts +32 -0
- package/dist/cache/backends/index.d.ts.map +1 -0
- package/dist/cache/backends/index.js +76 -0
- package/dist/cache/backends/index.js.map +1 -0
- package/dist/cache/backends/local-fs.d.ts +31 -0
- package/dist/cache/backends/local-fs.d.ts.map +1 -0
- package/dist/cache/backends/local-fs.js +101 -0
- package/dist/cache/backends/local-fs.js.map +1 -0
- package/dist/cache/backends/memory.d.ts +19 -0
- package/dist/cache/backends/memory.d.ts.map +1 -0
- package/dist/cache/backends/memory.js +30 -0
- package/dist/cache/backends/memory.js.map +1 -0
- package/dist/cache/backends/r2.d.ts +51 -0
- package/dist/cache/backends/r2.d.ts.map +1 -0
- package/dist/cache/backends/r2.js +116 -0
- package/dist/cache/backends/r2.js.map +1 -0
- package/dist/cache/classification-cache.d.ts +23 -9
- package/dist/cache/classification-cache.d.ts.map +1 -1
- package/dist/cache/classification-cache.js +37 -64
- package/dist/cache/classification-cache.js.map +1 -1
- package/dist/cache/discovery-cache.d.ts +71 -0
- package/dist/cache/discovery-cache.d.ts.map +1 -0
- package/dist/cache/discovery-cache.js +95 -0
- package/dist/cache/discovery-cache.js.map +1 -0
- package/dist/cache/index.d.ts +4 -0
- package/dist/cache/index.d.ts.map +1 -1
- package/dist/cache/index.js +2 -0
- package/dist/cache/index.js.map +1 -1
- package/dist/cache/tree-cache.d.ts +111 -0
- package/dist/cache/tree-cache.d.ts.map +1 -0
- package/dist/cache/tree-cache.js +171 -0
- package/dist/cache/tree-cache.js.map +1 -0
- package/dist/cache/verify-cache.d.ts +60 -0
- package/dist/cache/verify-cache.d.ts.map +1 -0
- package/dist/cache/verify-cache.js +103 -0
- package/dist/cache/verify-cache.js.map +1 -0
- package/dist/cluster/cluster.d.ts +174 -0
- package/dist/cluster/cluster.d.ts.map +1 -0
- package/dist/cluster/cluster.js +392 -0
- package/dist/cluster/cluster.js.map +1 -0
- package/dist/cluster/index.d.ts +11 -0
- package/dist/cluster/index.d.ts.map +1 -0
- package/dist/cluster/index.js +10 -0
- package/dist/cluster/index.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -1
- package/dist/llm/batch-classifier.js +2 -2
- package/dist/llm/batch-classifier.js.map +1 -1
- package/dist/llm/call-logger.d.ts +36 -0
- package/dist/llm/call-logger.d.ts.map +1 -1
- package/dist/llm/call-logger.js +36 -0
- package/dist/llm/call-logger.js.map +1 -1
- package/dist/llm/discovery.d.ts +1 -1
- package/dist/llm/discovery.d.ts.map +1 -1
- package/dist/llm/discovery.js +39 -0
- package/dist/llm/discovery.js.map +1 -1
- package/dist/llm/verification.d.ts +1 -1
- package/dist/llm/verification.d.ts.map +1 -1
- package/dist/llm/verification.js +37 -0
- package/dist/llm/verification.js.map +1 -1
- package/package.json +2 -1
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cache backend abstraction (cognium-ai#158 Acceptance #4).
|
|
3
|
+
*
|
|
4
|
+
* Defines the storage interface used by `ClassificationCache` so the
|
|
5
|
+
* three caches (tree / verify / discovery) can be persisted to any of:
|
|
6
|
+
* - local filesystem (default; preserves ≤2.26.0 behavior)
|
|
7
|
+
* - in-memory map (unit tests)
|
|
8
|
+
* - S3-compatible object store (Cloudflare R2, AWS S3, MinIO)
|
|
9
|
+
*
|
|
10
|
+
* The interface is byte-oriented and async — backends serialize raw
|
|
11
|
+
* UTF-8 JSON strings under opaque keys. TTL / hit-counter / fingerprint
|
|
12
|
+
* key composition stays in `ClassificationCache` so a backend swap
|
|
13
|
+
* doesn't change observable cache semantics.
|
|
14
|
+
*
|
|
15
|
+
* Selection: `LLM_CACHE_BACKEND` env (`local` | `memory` | `r2`).
|
|
16
|
+
* Default = `local` (back-compat). See `./backends/index.ts` factory.
|
|
17
|
+
*/
|
|
18
|
+
export interface CacheBackend {
|
|
19
|
+
/**
|
|
20
|
+
* Fetch the serialized payload for `key`, or null if absent.
|
|
21
|
+
* Backends MUST NOT treat "not found" as an error.
|
|
22
|
+
* Read errors (I/O, decode) MUST be swallowed and returned as null
|
|
23
|
+
* so the cache layer falls through to a miss instead of crashing
|
|
24
|
+
* the scan.
|
|
25
|
+
*/
|
|
26
|
+
get(key: string): Promise<string | null>;
|
|
27
|
+
/**
|
|
28
|
+
* Persist `value` under `key`. Backends MUST be safe under
|
|
29
|
+
* concurrent writers — local-fs uses tmp + rename, S3 backends
|
|
30
|
+
* rely on the object store's last-writer-wins semantics.
|
|
31
|
+
* Write errors MUST be swallowed (the cache is an optimization,
|
|
32
|
+
* not a correctness boundary).
|
|
33
|
+
*/
|
|
34
|
+
set(key: string, value: string): Promise<void>;
|
|
35
|
+
/** Optional: cheap existence check. Default impl is `get(...) !== null`. */
|
|
36
|
+
has?(key: string): Promise<boolean>;
|
|
37
|
+
/** Optional: delete a single entry. Used by TTL eviction. */
|
|
38
|
+
delete?(key: string): Promise<void>;
|
|
39
|
+
/**
|
|
40
|
+
* Optional: count / list entries. Used by `ClassificationCache.stats()`.
|
|
41
|
+
* S3 backends MAY be slow here; callers should treat as best-effort.
|
|
42
|
+
*/
|
|
43
|
+
count?(): Promise<number>;
|
|
44
|
+
/**
|
|
45
|
+
* Optional: drop everything. Used by tests; production callers
|
|
46
|
+
* should never hit this.
|
|
47
|
+
*/
|
|
48
|
+
clear?(): Promise<void>;
|
|
49
|
+
/** Human-readable backend identifier for logs (`local-fs`, `r2`, etc.). */
|
|
50
|
+
readonly kind: string;
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=backend.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"backend.d.ts","sourceRoot":"","sources":["../../src/cache/backend.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,MAAM,WAAW,YAAY;IAC3B;;;;;;OAMG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAEzC;;;;;;OAMG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE/C,4EAA4E;IAC5E,GAAG,CAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAEpC,6DAA6D;IAC7D,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEpC;;;OAGG;IACH,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAE1B;;;OAGG;IACH,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAExB,2EAA2E;IAC3E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cache backend abstraction (cognium-ai#158 Acceptance #4).
|
|
3
|
+
*
|
|
4
|
+
* Defines the storage interface used by `ClassificationCache` so the
|
|
5
|
+
* three caches (tree / verify / discovery) can be persisted to any of:
|
|
6
|
+
* - local filesystem (default; preserves ≤2.26.0 behavior)
|
|
7
|
+
* - in-memory map (unit tests)
|
|
8
|
+
* - S3-compatible object store (Cloudflare R2, AWS S3, MinIO)
|
|
9
|
+
*
|
|
10
|
+
* The interface is byte-oriented and async — backends serialize raw
|
|
11
|
+
* UTF-8 JSON strings under opaque keys. TTL / hit-counter / fingerprint
|
|
12
|
+
* key composition stays in `ClassificationCache` so a backend swap
|
|
13
|
+
* doesn't change observable cache semantics.
|
|
14
|
+
*
|
|
15
|
+
* Selection: `LLM_CACHE_BACKEND` env (`local` | `memory` | `r2`).
|
|
16
|
+
* Default = `local` (back-compat). See `./backends/index.ts` factory.
|
|
17
|
+
*/
|
|
18
|
+
export {};
|
|
19
|
+
//# sourceMappingURL=backend.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"backend.js","sourceRoot":"","sources":["../../src/cache/backend.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cache backend factory (cognium-ai#158 Acceptance #4).
|
|
3
|
+
*
|
|
4
|
+
* Selects a `CacheBackend` per cache-dir based on `LLM_CACHE_BACKEND`:
|
|
5
|
+
* `local` (default) — `LocalFSBackend(cacheDir)`
|
|
6
|
+
* `memory` — `MemoryBackend()`
|
|
7
|
+
* `r2` — `R2Backend({R2_ENDPOINT, R2_BUCKET, …})`
|
|
8
|
+
*
|
|
9
|
+
* For R2, the `cacheDir` is mapped to a stable sub-prefix so the three
|
|
10
|
+
* caches (tree / verify / discovery) live under separate keyspaces in
|
|
11
|
+
* a single bucket:
|
|
12
|
+
* tree → `<R2_PREFIX>/tree/`
|
|
13
|
+
* verify → `<R2_PREFIX>/verify/`
|
|
14
|
+
* discovery → `<R2_PREFIX>/discovery/`
|
|
15
|
+
* <other> → `<R2_PREFIX>/misc/`
|
|
16
|
+
*
|
|
17
|
+
* (Phase 2 #158 layout — `<lang>/_tree/<pkg>@<ver>/…` — lands as a
|
|
18
|
+
* separate refactor on top of this.)
|
|
19
|
+
*/
|
|
20
|
+
import type { CacheBackend } from '../backend.js';
|
|
21
|
+
import { LocalFSBackend } from './local-fs.js';
|
|
22
|
+
import { MemoryBackend } from './memory.js';
|
|
23
|
+
import { R2Backend } from './r2.js';
|
|
24
|
+
export { LocalFSBackend, MemoryBackend, R2Backend };
|
|
25
|
+
export declare function getBackendKind(): 'local' | 'memory' | 'r2';
|
|
26
|
+
/**
|
|
27
|
+
* Build the backend for a given cacheDir. `cacheDir` is the local-fs
|
|
28
|
+
* path the caller would have used pre-abstraction; for non-fs backends
|
|
29
|
+
* we use only its basename as a sub-prefix.
|
|
30
|
+
*/
|
|
31
|
+
export declare function makeBackend(cacheDir: string): CacheBackend;
|
|
32
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/cache/backends/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAEpC,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC;AAEpD,wBAAgB,cAAc,IAAI,OAAO,GAAG,QAAQ,GAAG,IAAI,CAK1D;AAED;;;;GAIG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,YAAY,CAkC1D"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cache backend factory (cognium-ai#158 Acceptance #4).
|
|
3
|
+
*
|
|
4
|
+
* Selects a `CacheBackend` per cache-dir based on `LLM_CACHE_BACKEND`:
|
|
5
|
+
* `local` (default) — `LocalFSBackend(cacheDir)`
|
|
6
|
+
* `memory` — `MemoryBackend()`
|
|
7
|
+
* `r2` — `R2Backend({R2_ENDPOINT, R2_BUCKET, …})`
|
|
8
|
+
*
|
|
9
|
+
* For R2, the `cacheDir` is mapped to a stable sub-prefix so the three
|
|
10
|
+
* caches (tree / verify / discovery) live under separate keyspaces in
|
|
11
|
+
* a single bucket:
|
|
12
|
+
* tree → `<R2_PREFIX>/tree/`
|
|
13
|
+
* verify → `<R2_PREFIX>/verify/`
|
|
14
|
+
* discovery → `<R2_PREFIX>/discovery/`
|
|
15
|
+
* <other> → `<R2_PREFIX>/misc/`
|
|
16
|
+
*
|
|
17
|
+
* (Phase 2 #158 layout — `<lang>/_tree/<pkg>@<ver>/…` — lands as a
|
|
18
|
+
* separate refactor on top of this.)
|
|
19
|
+
*/
|
|
20
|
+
import * as path from 'path';
|
|
21
|
+
import { LocalFSBackend } from './local-fs.js';
|
|
22
|
+
import { MemoryBackend } from './memory.js';
|
|
23
|
+
import { R2Backend } from './r2.js';
|
|
24
|
+
export { LocalFSBackend, MemoryBackend, R2Backend };
|
|
25
|
+
export function getBackendKind() {
|
|
26
|
+
const raw = (process.env.LLM_CACHE_BACKEND ?? '').toLowerCase().trim();
|
|
27
|
+
if (raw === 'r2')
|
|
28
|
+
return 'r2';
|
|
29
|
+
if (raw === 'memory')
|
|
30
|
+
return 'memory';
|
|
31
|
+
return 'local';
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Build the backend for a given cacheDir. `cacheDir` is the local-fs
|
|
35
|
+
* path the caller would have used pre-abstraction; for non-fs backends
|
|
36
|
+
* we use only its basename as a sub-prefix.
|
|
37
|
+
*/
|
|
38
|
+
export function makeBackend(cacheDir) {
|
|
39
|
+
const kind = getBackendKind();
|
|
40
|
+
if (kind === 'memory')
|
|
41
|
+
return new MemoryBackend();
|
|
42
|
+
if (kind === 'r2') {
|
|
43
|
+
const endpoint = process.env.R2_ENDPOINT;
|
|
44
|
+
const bucket = process.env.R2_BUCKET;
|
|
45
|
+
const accessKeyId = process.env.R2_ACCESS_KEY_ID;
|
|
46
|
+
const secretAccessKey = process.env.R2_SECRET_ACCESS_KEY;
|
|
47
|
+
if (!endpoint || !bucket || !accessKeyId || !secretAccessKey) {
|
|
48
|
+
// Misconfigured; degrade to local instead of crashing. The
|
|
49
|
+
// operator will see no cross-runner sharing but the scan
|
|
50
|
+
// still works.
|
|
51
|
+
// eslint-disable-next-line no-console
|
|
52
|
+
console.warn('[cache] LLM_CACHE_BACKEND=r2 but R2_ENDPOINT / R2_BUCKET / ' +
|
|
53
|
+
'R2_ACCESS_KEY_ID / R2_SECRET_ACCESS_KEY not all set — ' +
|
|
54
|
+
'falling back to local filesystem cache.');
|
|
55
|
+
return new LocalFSBackend({ cacheDir });
|
|
56
|
+
}
|
|
57
|
+
const userPrefix = (process.env.R2_PREFIX ?? 'v1').replace(/^\/+|\/+$/g, '');
|
|
58
|
+
const sub = cacheDirToSubPrefix(cacheDir);
|
|
59
|
+
return new R2Backend({
|
|
60
|
+
endpoint,
|
|
61
|
+
bucket,
|
|
62
|
+
accessKeyId,
|
|
63
|
+
secretAccessKey,
|
|
64
|
+
region: process.env.R2_REGION ?? 'auto',
|
|
65
|
+
prefix: `${userPrefix}/${sub}`,
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
return new LocalFSBackend({ cacheDir });
|
|
69
|
+
}
|
|
70
|
+
function cacheDirToSubPrefix(cacheDir) {
|
|
71
|
+
const base = path.basename(cacheDir).toLowerCase();
|
|
72
|
+
if (base === 'tree' || base === 'verify' || base === 'discovery')
|
|
73
|
+
return base;
|
|
74
|
+
return 'misc';
|
|
75
|
+
}
|
|
76
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/cache/backends/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAEpC,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC;AAEpD,MAAM,UAAU,cAAc;IAC5B,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;IACvE,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAC9B,IAAI,GAAG,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC;IACtC,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,QAAgB;IAC1C,MAAM,IAAI,GAAG,cAAc,EAAE,CAAC;IAC9B,IAAI,IAAI,KAAK,QAAQ;QAAE,OAAO,IAAI,aAAa,EAAE,CAAC;IAElD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC;QACzC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;QACrC,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;QACjD,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;QACzD,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM,IAAI,CAAC,WAAW,IAAI,CAAC,eAAe,EAAE,CAAC;YAC7D,2DAA2D;YAC3D,yDAAyD;YACzD,eAAe;YACf,sCAAsC;YACtC,OAAO,CAAC,IAAI,CACV,6DAA6D;gBAC7D,wDAAwD;gBACxD,yCAAyC,CAC1C,CAAC;YACF,OAAO,IAAI,cAAc,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC1C,CAAC;QACD,MAAM,UAAU,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QAC7E,MAAM,GAAG,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QAC1C,OAAO,IAAI,SAAS,CAAC;YACnB,QAAQ;YACR,MAAM;YACN,WAAW;YACX,eAAe;YACf,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM;YACvC,MAAM,EAAE,GAAG,UAAU,IAAI,GAAG,EAAE;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,OAAO,IAAI,cAAc,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;AAC1C,CAAC;AAED,SAAS,mBAAmB,CAAC,QAAgB;IAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IACnD,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,WAAW;QAAE,OAAO,IAAI,CAAC;IAC9E,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local filesystem cache backend (cognium-ai#158 Acceptance #4).
|
|
3
|
+
*
|
|
4
|
+
* Preserves ≤2.26.0 on-disk shape:
|
|
5
|
+
* <cacheDir>/<key[0:2]>/<key>.json
|
|
6
|
+
*
|
|
7
|
+
* Atomic write via tmp + rename so concurrent `cognium-ai scan`
|
|
8
|
+
* processes don't corrupt each other's entries (same pattern as
|
|
9
|
+
* the previous inline implementation in `classification-cache.ts`).
|
|
10
|
+
*
|
|
11
|
+
* Read/write errors are swallowed and surfaced as cache misses.
|
|
12
|
+
*/
|
|
13
|
+
import type { CacheBackend } from '../backend.js';
|
|
14
|
+
export interface LocalFSBackendOptions {
|
|
15
|
+
/** Root directory under which `<key[0:2]>/<key>.json` lives. */
|
|
16
|
+
cacheDir: string;
|
|
17
|
+
}
|
|
18
|
+
export declare class LocalFSBackend implements CacheBackend {
|
|
19
|
+
readonly kind = "local-fs";
|
|
20
|
+
private cacheDir;
|
|
21
|
+
constructor(opts: LocalFSBackendOptions);
|
|
22
|
+
private ensureCacheDir;
|
|
23
|
+
private cachePath;
|
|
24
|
+
get(key: string): Promise<string | null>;
|
|
25
|
+
set(key: string, value: string): Promise<void>;
|
|
26
|
+
has(key: string): Promise<boolean>;
|
|
27
|
+
delete(key: string): Promise<void>;
|
|
28
|
+
count(): Promise<number>;
|
|
29
|
+
clear(): Promise<void>;
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=local-fs.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"local-fs.d.ts","sourceRoot":"","sources":["../../../src/cache/backends/local-fs.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAElD,MAAM,WAAW,qBAAqB;IACpC,gEAAgE;IAChE,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,cAAe,YAAW,YAAY;IACjD,QAAQ,CAAC,IAAI,cAAc;IAC3B,OAAO,CAAC,QAAQ,CAAS;gBAEb,IAAI,EAAE,qBAAqB;IAKvC,OAAO,CAAC,cAAc;IAMtB,OAAO,CAAC,SAAS;IAKX,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAUxC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAe9C,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAIlC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASlC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;IAgBxB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAQ7B"}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local filesystem cache backend (cognium-ai#158 Acceptance #4).
|
|
3
|
+
*
|
|
4
|
+
* Preserves ≤2.26.0 on-disk shape:
|
|
5
|
+
* <cacheDir>/<key[0:2]>/<key>.json
|
|
6
|
+
*
|
|
7
|
+
* Atomic write via tmp + rename so concurrent `cognium-ai scan`
|
|
8
|
+
* processes don't corrupt each other's entries (same pattern as
|
|
9
|
+
* the previous inline implementation in `classification-cache.ts`).
|
|
10
|
+
*
|
|
11
|
+
* Read/write errors are swallowed and surfaced as cache misses.
|
|
12
|
+
*/
|
|
13
|
+
import * as fs from 'fs';
|
|
14
|
+
import * as path from 'path';
|
|
15
|
+
export class LocalFSBackend {
|
|
16
|
+
kind = 'local-fs';
|
|
17
|
+
cacheDir;
|
|
18
|
+
constructor(opts) {
|
|
19
|
+
this.cacheDir = opts.cacheDir;
|
|
20
|
+
this.ensureCacheDir();
|
|
21
|
+
}
|
|
22
|
+
ensureCacheDir() {
|
|
23
|
+
if (!fs.existsSync(this.cacheDir)) {
|
|
24
|
+
fs.mkdirSync(this.cacheDir, { recursive: true });
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
cachePath(key) {
|
|
28
|
+
const subdir = key.substring(0, 2);
|
|
29
|
+
return path.join(this.cacheDir, subdir, `${key}.json`);
|
|
30
|
+
}
|
|
31
|
+
async get(key) {
|
|
32
|
+
const p = this.cachePath(key);
|
|
33
|
+
try {
|
|
34
|
+
if (!fs.existsSync(p))
|
|
35
|
+
return null;
|
|
36
|
+
return fs.readFileSync(p, 'utf-8');
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
return null;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
async set(key, value) {
|
|
43
|
+
const p = this.cachePath(key);
|
|
44
|
+
const subdir = path.dirname(p);
|
|
45
|
+
const tmp = `${p}.tmp.${process.pid}.${Math.random().toString(36).slice(2, 8)}`;
|
|
46
|
+
try {
|
|
47
|
+
if (!fs.existsSync(subdir)) {
|
|
48
|
+
fs.mkdirSync(subdir, { recursive: true });
|
|
49
|
+
}
|
|
50
|
+
fs.writeFileSync(tmp, value);
|
|
51
|
+
fs.renameSync(tmp, p);
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
try {
|
|
55
|
+
fs.unlinkSync(tmp);
|
|
56
|
+
}
|
|
57
|
+
catch { /* ignore */ }
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
async has(key) {
|
|
61
|
+
return fs.existsSync(this.cachePath(key));
|
|
62
|
+
}
|
|
63
|
+
async delete(key) {
|
|
64
|
+
const p = this.cachePath(key);
|
|
65
|
+
try {
|
|
66
|
+
if (fs.existsSync(p))
|
|
67
|
+
fs.unlinkSync(p);
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
/* ignore */
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
async count() {
|
|
74
|
+
let n = 0;
|
|
75
|
+
if (!fs.existsSync(this.cacheDir))
|
|
76
|
+
return 0;
|
|
77
|
+
try {
|
|
78
|
+
for (const sub of fs.readdirSync(this.cacheDir)) {
|
|
79
|
+
const subPath = path.join(this.cacheDir, sub);
|
|
80
|
+
try {
|
|
81
|
+
if (fs.statSync(subPath).isDirectory()) {
|
|
82
|
+
n += fs.readdirSync(subPath).length;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
catch { /* skip */ }
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
catch { /* ignore */ }
|
|
89
|
+
return n;
|
|
90
|
+
}
|
|
91
|
+
async clear() {
|
|
92
|
+
try {
|
|
93
|
+
if (fs.existsSync(this.cacheDir)) {
|
|
94
|
+
fs.rmSync(this.cacheDir, { recursive: true, force: true });
|
|
95
|
+
}
|
|
96
|
+
this.ensureCacheDir();
|
|
97
|
+
}
|
|
98
|
+
catch { /* ignore */ }
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
//# sourceMappingURL=local-fs.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"local-fs.js","sourceRoot":"","sources":["../../../src/cache/backends/local-fs.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAQ7B,MAAM,OAAO,cAAc;IAChB,IAAI,GAAG,UAAU,CAAC;IACnB,QAAQ,CAAS;IAEzB,YAAY,IAA2B;QACrC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC9B,IAAI,CAAC,cAAc,EAAE,CAAC;IACxB,CAAC;IAEO,cAAc;QACpB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YAClC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAEO,SAAS,CAAC,GAAW;QAC3B,MAAM,MAAM,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACnC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;gBAAE,OAAO,IAAI,CAAC;YACnC,OAAO,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QACrC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW,EAAE,KAAa;QAClC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAChF,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3B,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC5C,CAAC;YACD,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAC7B,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACxB,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC;gBAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,OAAO,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC;YACH,IAAI,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;gBAAE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACzC,CAAC;QAAC,MAAM,CAAC;YACP,YAAY;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC;YAAE,OAAO,CAAC,CAAC;QAC5C,IAAI,CAAC;YACH,KAAK,MAAM,GAAG,IAAI,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAChD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;gBAC9C,IAAI,CAAC;oBACH,IAAI,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;wBACvC,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;oBACtC,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC,CAAC,UAAU,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;QACxB,OAAO,CAAC,CAAC;IACX,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC;YACH,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACjC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC7D,CAAC;YACD,IAAI,CAAC,cAAc,EAAE,CAAC;QACxB,CAAC;QAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IAC1B,CAAC;CACF"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* In-memory cache backend (cognium-ai#158 Acceptance #4).
|
|
3
|
+
*
|
|
4
|
+
* Map-backed; used by unit tests to avoid filesystem coupling.
|
|
5
|
+
* Not intended for production — no eviction, no persistence,
|
|
6
|
+
* grows unbounded.
|
|
7
|
+
*/
|
|
8
|
+
import type { CacheBackend } from '../backend.js';
|
|
9
|
+
export declare class MemoryBackend implements CacheBackend {
|
|
10
|
+
readonly kind = "memory";
|
|
11
|
+
private store;
|
|
12
|
+
get(key: string): Promise<string | null>;
|
|
13
|
+
set(key: string, value: string): Promise<void>;
|
|
14
|
+
has(key: string): Promise<boolean>;
|
|
15
|
+
delete(key: string): Promise<void>;
|
|
16
|
+
count(): Promise<number>;
|
|
17
|
+
clear(): Promise<void>;
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=memory.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../../src/cache/backends/memory.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAElD,qBAAa,aAAc,YAAW,YAAY;IAChD,QAAQ,CAAC,IAAI,YAAY;IACzB,OAAO,CAAC,KAAK,CAAkC;IAEzC,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAIxC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAI9C,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAIlC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIlC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;IAIxB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* In-memory cache backend (cognium-ai#158 Acceptance #4).
|
|
3
|
+
*
|
|
4
|
+
* Map-backed; used by unit tests to avoid filesystem coupling.
|
|
5
|
+
* Not intended for production — no eviction, no persistence,
|
|
6
|
+
* grows unbounded.
|
|
7
|
+
*/
|
|
8
|
+
export class MemoryBackend {
|
|
9
|
+
kind = 'memory';
|
|
10
|
+
store = new Map();
|
|
11
|
+
async get(key) {
|
|
12
|
+
return this.store.get(key) ?? null;
|
|
13
|
+
}
|
|
14
|
+
async set(key, value) {
|
|
15
|
+
this.store.set(key, value);
|
|
16
|
+
}
|
|
17
|
+
async has(key) {
|
|
18
|
+
return this.store.has(key);
|
|
19
|
+
}
|
|
20
|
+
async delete(key) {
|
|
21
|
+
this.store.delete(key);
|
|
22
|
+
}
|
|
23
|
+
async count() {
|
|
24
|
+
return this.store.size;
|
|
25
|
+
}
|
|
26
|
+
async clear() {
|
|
27
|
+
this.store.clear();
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
//# sourceMappingURL=memory.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory.js","sourceRoot":"","sources":["../../../src/cache/backends/memory.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,MAAM,OAAO,aAAa;IACf,IAAI,GAAG,QAAQ,CAAC;IACjB,KAAK,GAAwB,IAAI,GAAG,EAAE,CAAC;IAE/C,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW,EAAE,KAAa;QAClC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;CACF"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cloudflare R2 (S3-compatible) cache backend (cognium-ai#158).
|
|
3
|
+
*
|
|
4
|
+
* Phase 1 scope: drop-in object-store mirror of the local-fs key shape.
|
|
5
|
+
* The R2 object key is `<prefix>/<key[0:2]>/<key>.json` — same sharding
|
|
6
|
+
* as `LocalFSBackend` so the existing 3-component `ClassificationCache`
|
|
7
|
+
* keys (content + classifier + modelFingerprint hash) just work.
|
|
8
|
+
*
|
|
9
|
+
* Phase 2 (#158 layout reshape — separate follow-up): swap the opaque
|
|
10
|
+
* 64-hex key for the structured `<lang>/_tree/<pkg>@<ver>/<fileSHA>.json`
|
|
11
|
+
* style and reuse the `pkg-coord` + `model-registry` normalization
|
|
12
|
+
* helpers. Phase 1 only proves wire-up + cross-runner sharing.
|
|
13
|
+
*
|
|
14
|
+
* Configuration (env vars):
|
|
15
|
+
* LLM_CACHE_BACKEND=r2
|
|
16
|
+
* R2_ENDPOINT=https://<account-id>.r2.cloudflarestorage.com
|
|
17
|
+
* R2_BUCKET=cognium-cache
|
|
18
|
+
* R2_ACCESS_KEY_ID=<S3-compat access key>
|
|
19
|
+
* R2_SECRET_ACCESS_KEY=<S3-compat secret>
|
|
20
|
+
* R2_REGION=auto (default; R2 ignores region)
|
|
21
|
+
* R2_PREFIX=v1 (default; namespaces the bucket)
|
|
22
|
+
*
|
|
23
|
+
* Failure mode: R2 errors are swallowed and surfaced as misses so a
|
|
24
|
+
* network blip during scan degrades to local behavior, not a crash.
|
|
25
|
+
* This matches `LocalFSBackend` and the cache-as-optimization contract
|
|
26
|
+
* spelled out in `backend.ts`.
|
|
27
|
+
*/
|
|
28
|
+
import type { CacheBackend } from '../backend.js';
|
|
29
|
+
export interface R2BackendOptions {
|
|
30
|
+
endpoint: string;
|
|
31
|
+
bucket: string;
|
|
32
|
+
accessKeyId: string;
|
|
33
|
+
secretAccessKey: string;
|
|
34
|
+
/** R2 ignores region; default `auto`. */
|
|
35
|
+
region?: string;
|
|
36
|
+
/** Object-key prefix (default `v1`). Used to roll layout versions. */
|
|
37
|
+
prefix?: string;
|
|
38
|
+
}
|
|
39
|
+
export declare class R2Backend implements CacheBackend {
|
|
40
|
+
readonly kind = "r2";
|
|
41
|
+
private client;
|
|
42
|
+
private bucket;
|
|
43
|
+
private prefix;
|
|
44
|
+
constructor(opts: R2BackendOptions);
|
|
45
|
+
private objectKey;
|
|
46
|
+
get(key: string): Promise<string | null>;
|
|
47
|
+
set(key: string, value: string): Promise<void>;
|
|
48
|
+
has(key: string): Promise<boolean>;
|
|
49
|
+
delete(key: string): Promise<void>;
|
|
50
|
+
}
|
|
51
|
+
//# sourceMappingURL=r2.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"r2.d.ts","sourceRoot":"","sources":["../../../src/cache/backends/r2.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AASH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAElD,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,yCAAyC;IACzC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,sEAAsE;IACtE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,qBAAa,SAAU,YAAW,YAAY;IAC5C,QAAQ,CAAC,IAAI,QAAQ;IACrB,OAAO,CAAC,MAAM,CAAW;IACzB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,MAAM,CAAS;gBAEX,IAAI,EAAE,gBAAgB;IAelC,OAAO,CAAC,SAAS;IAKX,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAkBxC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAa9C,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAYlC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAUzC"}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cloudflare R2 (S3-compatible) cache backend (cognium-ai#158).
|
|
3
|
+
*
|
|
4
|
+
* Phase 1 scope: drop-in object-store mirror of the local-fs key shape.
|
|
5
|
+
* The R2 object key is `<prefix>/<key[0:2]>/<key>.json` — same sharding
|
|
6
|
+
* as `LocalFSBackend` so the existing 3-component `ClassificationCache`
|
|
7
|
+
* keys (content + classifier + modelFingerprint hash) just work.
|
|
8
|
+
*
|
|
9
|
+
* Phase 2 (#158 layout reshape — separate follow-up): swap the opaque
|
|
10
|
+
* 64-hex key for the structured `<lang>/_tree/<pkg>@<ver>/<fileSHA>.json`
|
|
11
|
+
* style and reuse the `pkg-coord` + `model-registry` normalization
|
|
12
|
+
* helpers. Phase 1 only proves wire-up + cross-runner sharing.
|
|
13
|
+
*
|
|
14
|
+
* Configuration (env vars):
|
|
15
|
+
* LLM_CACHE_BACKEND=r2
|
|
16
|
+
* R2_ENDPOINT=https://<account-id>.r2.cloudflarestorage.com
|
|
17
|
+
* R2_BUCKET=cognium-cache
|
|
18
|
+
* R2_ACCESS_KEY_ID=<S3-compat access key>
|
|
19
|
+
* R2_SECRET_ACCESS_KEY=<S3-compat secret>
|
|
20
|
+
* R2_REGION=auto (default; R2 ignores region)
|
|
21
|
+
* R2_PREFIX=v1 (default; namespaces the bucket)
|
|
22
|
+
*
|
|
23
|
+
* Failure mode: R2 errors are swallowed and surfaced as misses so a
|
|
24
|
+
* network blip during scan degrades to local behavior, not a crash.
|
|
25
|
+
* This matches `LocalFSBackend` and the cache-as-optimization contract
|
|
26
|
+
* spelled out in `backend.ts`.
|
|
27
|
+
*/
|
|
28
|
+
import { S3Client, GetObjectCommand, PutObjectCommand, HeadObjectCommand, DeleteObjectCommand, } from '@aws-sdk/client-s3';
|
|
29
|
+
export class R2Backend {
|
|
30
|
+
kind = 'r2';
|
|
31
|
+
client;
|
|
32
|
+
bucket;
|
|
33
|
+
prefix;
|
|
34
|
+
constructor(opts) {
|
|
35
|
+
this.bucket = opts.bucket;
|
|
36
|
+
this.prefix = (opts.prefix ?? 'v1').replace(/^\/+|\/+$/g, '');
|
|
37
|
+
this.client = new S3Client({
|
|
38
|
+
endpoint: opts.endpoint,
|
|
39
|
+
region: opts.region ?? 'auto',
|
|
40
|
+
credentials: {
|
|
41
|
+
accessKeyId: opts.accessKeyId,
|
|
42
|
+
secretAccessKey: opts.secretAccessKey,
|
|
43
|
+
},
|
|
44
|
+
// R2 requires path-style addressing for the S3-compat endpoint.
|
|
45
|
+
forcePathStyle: true,
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
objectKey(key) {
|
|
49
|
+
const sub = key.substring(0, 2);
|
|
50
|
+
return `${this.prefix}/${sub}/${key}.json`;
|
|
51
|
+
}
|
|
52
|
+
async get(key) {
|
|
53
|
+
try {
|
|
54
|
+
const out = await this.client.send(new GetObjectCommand({
|
|
55
|
+
Bucket: this.bucket,
|
|
56
|
+
Key: this.objectKey(key),
|
|
57
|
+
}));
|
|
58
|
+
if (!out.Body)
|
|
59
|
+
return null;
|
|
60
|
+
// SDK v3 returns a stream; collect to string.
|
|
61
|
+
const stream = out.Body;
|
|
62
|
+
return await streamToString(stream);
|
|
63
|
+
}
|
|
64
|
+
catch (err) {
|
|
65
|
+
// 404 → miss. Other errors swallowed (cache is best-effort).
|
|
66
|
+
const name = err?.name;
|
|
67
|
+
if (name === 'NoSuchKey' || name === 'NotFound')
|
|
68
|
+
return null;
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
async set(key, value) {
|
|
73
|
+
try {
|
|
74
|
+
await this.client.send(new PutObjectCommand({
|
|
75
|
+
Bucket: this.bucket,
|
|
76
|
+
Key: this.objectKey(key),
|
|
77
|
+
Body: value,
|
|
78
|
+
ContentType: 'application/json',
|
|
79
|
+
}));
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
/* swallow: cache write failure is non-fatal */
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
async has(key) {
|
|
86
|
+
try {
|
|
87
|
+
await this.client.send(new HeadObjectCommand({
|
|
88
|
+
Bucket: this.bucket,
|
|
89
|
+
Key: this.objectKey(key),
|
|
90
|
+
}));
|
|
91
|
+
return true;
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
return false;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
async delete(key) {
|
|
98
|
+
try {
|
|
99
|
+
await this.client.send(new DeleteObjectCommand({
|
|
100
|
+
Bucket: this.bucket,
|
|
101
|
+
Key: this.objectKey(key),
|
|
102
|
+
}));
|
|
103
|
+
}
|
|
104
|
+
catch {
|
|
105
|
+
/* swallow */
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
async function streamToString(stream) {
|
|
110
|
+
const chunks = [];
|
|
111
|
+
for await (const chunk of stream) {
|
|
112
|
+
chunks.push(typeof chunk === 'string' ? Buffer.from(chunk) : chunk);
|
|
113
|
+
}
|
|
114
|
+
return Buffer.concat(chunks).toString('utf-8');
|
|
115
|
+
}
|
|
116
|
+
//# sourceMappingURL=r2.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"r2.js","sourceRoot":"","sources":["../../../src/cache/backends/r2.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EACL,QAAQ,EACR,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,GACpB,MAAM,oBAAoB,CAAC;AAc5B,MAAM,OAAO,SAAS;IACX,IAAI,GAAG,IAAI,CAAC;IACb,MAAM,CAAW;IACjB,MAAM,CAAS;IACf,MAAM,CAAS;IAEvB,YAAY,IAAsB;QAChC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QAC9D,IAAI,CAAC,MAAM,GAAG,IAAI,QAAQ,CAAC;YACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,MAAM;YAC7B,WAAW,EAAE;gBACX,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,eAAe,EAAE,IAAI,CAAC,eAAe;aACtC;YACD,gEAAgE;YAChE,cAAc,EAAE,IAAI;SACrB,CAAC,CAAC;IACL,CAAC;IAEO,SAAS,CAAC,GAAW;QAC3B,MAAM,GAAG,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAChC,OAAO,GAAG,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,GAAG,OAAO,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC;gBACtD,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;aACzB,CAAC,CAAC,CAAC;YACJ,IAAI,CAAC,GAAG,CAAC,IAAI;gBAAE,OAAO,IAAI,CAAC;YAC3B,8CAA8C;YAC9C,MAAM,MAAM,GAAG,GAAG,CAAC,IAA6B,CAAC;YACjD,OAAO,MAAM,cAAc,CAAC,MAAM,CAAC,CAAC;QACtC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,6DAA6D;YAC7D,MAAM,IAAI,GAAI,GAAyB,EAAE,IAAI,CAAC;YAC9C,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,UAAU;gBAAE,OAAO,IAAI,CAAC;YAC7D,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW,EAAE,KAAa;QAClC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC;gBAC1C,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;gBACxB,IAAI,EAAE,KAAK;gBACX,WAAW,EAAE,kBAAkB;aAChC,CAAC,CAAC,CAAC;QACN,CAAC;QAAC,MAAM,CAAC;YACP,+CAA+C;QACjD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,iBAAiB,CAAC;gBAC3C,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;aACzB,CAAC,CAAC,CAAC;YACJ,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,mBAAmB,CAAC;gBAC7C,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;aACzB,CAAC,CAAC,CAAC;QACN,CAAC;QAAC,MAAM,CAAC;YACP,aAAa;QACf,CAAC;IACH,CAAC;CACF;AAED,KAAK,UAAU,cAAc,CAAC,MAA6B;IACzD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAE,KAAgB,CAAC,CAAC;IAClF,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;AACjD,CAAC"}
|
|
@@ -11,6 +11,11 @@
|
|
|
11
11
|
* - Cache directory: .cognium/cache/
|
|
12
12
|
* - Hit/miss tracking for observability
|
|
13
13
|
*
|
|
14
|
+
* Storage is delegated to a `CacheBackend` (cognium-ai#158 Acceptance #4).
|
|
15
|
+
* Default = `LocalFSBackend` (preserves ≤2.26.0 on-disk shape). Selecting
|
|
16
|
+
* `LLM_CACHE_BACKEND=r2` swaps to Cloudflare R2 transparently so two
|
|
17
|
+
* runners on different machines share the same cache.
|
|
18
|
+
*
|
|
14
19
|
* Model dimension (cognium-ai#151 P0 spike):
|
|
15
20
|
* When `modelFingerprint` is supplied (typically via
|
|
16
21
|
* `getPhaseModelFingerprint()` from `../llm/config.ts`) it becomes part
|
|
@@ -24,8 +29,10 @@
|
|
|
24
29
|
* ≤2.21.0) keep working — the first run after wire-through is a cache
|
|
25
30
|
* miss only for the call sites that opt into the fingerprint.
|
|
26
31
|
*/
|
|
32
|
+
import type { CacheBackend } from './backend.js';
|
|
27
33
|
export interface ClassificationCacheOptions {
|
|
28
|
-
/** Cache directory (default: .cognium/cache/ in project root)
|
|
34
|
+
/** Cache directory (default: .cognium/cache/ in project root). For
|
|
35
|
+
* non-fs backends this is interpreted as a sub-prefix only. */
|
|
29
36
|
cacheDir?: string;
|
|
30
37
|
/** Maximum age of entries in ms (default: 24h) */
|
|
31
38
|
maxAge?: number;
|
|
@@ -39,31 +46,38 @@ export interface ClassificationCacheOptions {
|
|
|
39
46
|
* Defaults to empty string (legacy 2-component key shape).
|
|
40
47
|
*/
|
|
41
48
|
modelFingerprint?: string;
|
|
49
|
+
/**
|
|
50
|
+
* Override the backend instance (tests). When omitted we build one
|
|
51
|
+
* via `makeBackend(cacheDir)` based on `LLM_CACHE_BACKEND` env.
|
|
52
|
+
*/
|
|
53
|
+
backend?: CacheBackend;
|
|
42
54
|
}
|
|
43
55
|
export declare class ClassificationCache<T = unknown> {
|
|
44
56
|
private cacheDir;
|
|
45
57
|
private maxAge;
|
|
46
58
|
private enabled;
|
|
47
59
|
private modelFingerprint;
|
|
60
|
+
private backend;
|
|
48
61
|
private memoryCache;
|
|
49
62
|
private hits;
|
|
50
63
|
private misses;
|
|
51
64
|
constructor(options?: ClassificationCacheOptions);
|
|
52
65
|
/** Per-phase model identity used in cache key (cognium-ai#151). */
|
|
53
66
|
getModelFingerprint(): string;
|
|
67
|
+
/** Backend kind identifier (`local-fs` | `memory` | `r2`). Observability. */
|
|
68
|
+
getBackendKind(): string;
|
|
54
69
|
private computeKey;
|
|
55
|
-
private getCachePath;
|
|
56
|
-
private ensureCacheDir;
|
|
57
70
|
private isValid;
|
|
58
|
-
get(content: string, classifier: string): T | null
|
|
59
|
-
set(content: string, classifier: string, result: T): void
|
|
60
|
-
has(content: string, classifier: string): boolean
|
|
61
|
-
clear(): void
|
|
62
|
-
stats(): {
|
|
71
|
+
get(content: string, classifier: string): Promise<T | null>;
|
|
72
|
+
set(content: string, classifier: string, result: T): Promise<void>;
|
|
73
|
+
has(content: string, classifier: string): Promise<boolean>;
|
|
74
|
+
clear(): Promise<void>;
|
|
75
|
+
stats(): Promise<{
|
|
63
76
|
memoryEntries: number;
|
|
64
77
|
diskEntries: number;
|
|
65
78
|
hitRate: number;
|
|
66
|
-
|
|
79
|
+
backend: string;
|
|
80
|
+
}>;
|
|
67
81
|
}
|
|
68
82
|
export declare function getClassificationCache(options?: ClassificationCacheOptions): ClassificationCache;
|
|
69
83
|
export declare function resetClassificationCache(): void;
|