@untangle-code/storage-local 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/NOTICE +90 -0
- package/README.md +226 -2
- package/THIRD_PARTY_LICENSES.md +24 -0
- package/dist/abort.d.ts +18 -0
- package/dist/abort.d.ts.map +1 -0
- package/dist/abort.js +14 -0
- package/dist/abort.js.map +1 -0
- package/dist/analysis-cache.d.ts +81 -0
- package/dist/analysis-cache.d.ts.map +1 -0
- package/dist/analysis-cache.js +322 -0
- package/dist/analysis-cache.js.map +1 -0
- package/dist/checkpoint-store.d.ts +22 -0
- package/dist/checkpoint-store.d.ts.map +1 -0
- package/dist/checkpoint-store.js +123 -0
- package/dist/checkpoint-store.js.map +1 -0
- package/dist/embedding-cache.d.ts +38 -0
- package/dist/embedding-cache.d.ts.map +1 -0
- package/dist/embedding-cache.js +229 -0
- package/dist/embedding-cache.js.map +1 -0
- package/dist/fs-errors.d.ts +4 -0
- package/dist/fs-errors.d.ts.map +1 -0
- package/dist/fs-errors.js +14 -0
- package/dist/fs-errors.js.map +1 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +23 -0
- package/dist/index.js.map +1 -0
- package/dist/layout.d.ts +50 -0
- package/dist/layout.d.ts.map +1 -0
- package/dist/layout.js +99 -0
- package/dist/layout.js.map +1 -0
- package/dist/local-file-system.d.ts +37 -0
- package/dist/local-file-system.d.ts.map +1 -0
- package/dist/local-file-system.js +193 -0
- package/dist/local-file-system.js.map +1 -0
- package/dist/lock-provider.d.ts +59 -0
- package/dist/lock-provider.d.ts.map +1 -0
- package/dist/lock-provider.js +318 -0
- package/dist/lock-provider.js.map +1 -0
- package/dist/real-time.d.ts +16 -0
- package/dist/real-time.d.ts.map +1 -0
- package/dist/real-time.js +19 -0
- package/dist/real-time.js.map +1 -0
- package/dist/search-journey-store.d.ts +34 -0
- package/dist/search-journey-store.d.ts.map +1 -0
- package/dist/search-journey-store.js +530 -0
- package/dist/search-journey-store.js.map +1 -0
- package/dist/storage.d.ts +42 -0
- package/dist/storage.d.ts.map +1 -0
- package/dist/storage.js +34 -0
- package/dist/storage.js.map +1 -0
- package/package.json +60 -3
- package/src/abort.ts +26 -0
- package/src/analysis-cache.ts +404 -0
- package/src/checkpoint-store.ts +155 -0
- package/src/embedding-cache.ts +282 -0
- package/src/fs-errors.ts +19 -0
- package/src/index.ts +30 -0
- package/src/layout.ts +149 -0
- package/src/local-file-system.ts +232 -0
- package/src/lock-provider.ts +381 -0
- package/src/real-time.ts +18 -0
- package/src/search-journey-store.ts +638 -0
- package/src/storage.ts +98 -0
package/src/storage.ts
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Convenience wiring: one call that builds the `.untangle/` layout, ensures
|
|
3
|
+
* every directory exists, and hands back the adapters this package provides
|
|
4
|
+
* — a `FileSystemPort` sandboxed to that state root, the embedding cache,
|
|
5
|
+
* the three content-addressed analysis caches (WP-100), the checkpoint mirror
|
|
6
|
+
* and the file-based lock provider, all sharing the same underlying
|
|
7
|
+
* `FileSystemPort`.
|
|
8
|
+
*/
|
|
9
|
+
import type {
|
|
10
|
+
Clock,
|
|
11
|
+
FileSystemPort,
|
|
12
|
+
LockProvider,
|
|
13
|
+
PortCallOptions,
|
|
14
|
+
Random,
|
|
15
|
+
} from "@untangle-code/domain";
|
|
16
|
+
|
|
17
|
+
import {
|
|
18
|
+
DEFAULT_ANALYSIS_CACHE_MAX_BYTES,
|
|
19
|
+
LocalAnalysisCache,
|
|
20
|
+
LocalContentCache,
|
|
21
|
+
} from "./analysis-cache.js";
|
|
22
|
+
import { LocalCheckpointStore } from "./checkpoint-store.js";
|
|
23
|
+
import { DEFAULT_EMBEDDING_CACHE_MAX_BYTES, LocalEmbeddingCache } from "./embedding-cache.js";
|
|
24
|
+
import { buildStateDirLayout, ensureStateDirLayout, type StateDirLayout } from "./layout.js";
|
|
25
|
+
import { LocalFileSystem } from "./local-file-system.js";
|
|
26
|
+
import { FileLockProvider } from "./lock-provider.js";
|
|
27
|
+
import { LocalSearchJourneyStore } from "./search-journey-store.js";
|
|
28
|
+
|
|
29
|
+
export interface LocalStorageOptions {
|
|
30
|
+
clock: Clock;
|
|
31
|
+
random: Random;
|
|
32
|
+
/** Overrides the persistent embedding cache's size limit (bytes). */
|
|
33
|
+
embeddingCacheMaxBytes?: number;
|
|
34
|
+
/** Overrides the size limit (bytes) of EACH of the three analysis caches (WP-100). */
|
|
35
|
+
analysisCacheMaxBytes?: number;
|
|
36
|
+
/** Overrides `os.hostname()` for lock records — mainly for tests. */
|
|
37
|
+
host?: string;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface LocalStorage {
|
|
41
|
+
layout: StateDirLayout;
|
|
42
|
+
fs: FileSystemPort;
|
|
43
|
+
embeddingCache: LocalEmbeddingCache;
|
|
44
|
+
/** Parse-, Chunk- und Metrik-Cache, inhaltsadressiert (WP-100). */
|
|
45
|
+
analysisCache: LocalAnalysisCache;
|
|
46
|
+
checkpoints: LocalCheckpointStore;
|
|
47
|
+
locks: LockProvider;
|
|
48
|
+
searchJourneys: LocalSearchJourneyStore;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Builds the `.untangle/` layout under `stateRoot` (creating every directory
|
|
53
|
+
* if needed) and returns the wired-up local storage adapters. `stateRoot`
|
|
54
|
+
* becomes the ONLY root the returned `FileSystemPort` will resolve paths
|
|
55
|
+
* within (rule 3: "kein Schreiben ausserhalb des Zustandsverzeichnisses").
|
|
56
|
+
*/
|
|
57
|
+
export async function openLocalStorage(
|
|
58
|
+
stateRoot: string,
|
|
59
|
+
options: LocalStorageOptions,
|
|
60
|
+
callOptions?: PortCallOptions,
|
|
61
|
+
): Promise<LocalStorage> {
|
|
62
|
+
const fs = new LocalFileSystem([stateRoot]);
|
|
63
|
+
const layout = buildStateDirLayout(stateRoot);
|
|
64
|
+
await ensureStateDirLayout(fs, layout, callOptions);
|
|
65
|
+
|
|
66
|
+
const embeddingCache = new LocalEmbeddingCache(
|
|
67
|
+
fs,
|
|
68
|
+
layout.cache.embeddings,
|
|
69
|
+
options.clock,
|
|
70
|
+
options.random,
|
|
71
|
+
{
|
|
72
|
+
maxBytes: options.embeddingCacheMaxBytes ?? DEFAULT_EMBEDDING_CACHE_MAX_BYTES,
|
|
73
|
+
},
|
|
74
|
+
);
|
|
75
|
+
const analysisMaxBytes = options.analysisCacheMaxBytes ?? DEFAULT_ANALYSIS_CACHE_MAX_BYTES;
|
|
76
|
+
const analysisCache = new LocalAnalysisCache(
|
|
77
|
+
new LocalContentCache(fs, layout.cache.analysis.parse, options.clock, options.random, {
|
|
78
|
+
maxBytes: analysisMaxBytes,
|
|
79
|
+
}),
|
|
80
|
+
new LocalContentCache(fs, layout.cache.analysis.chunk, options.clock, options.random, {
|
|
81
|
+
maxBytes: analysisMaxBytes,
|
|
82
|
+
}),
|
|
83
|
+
new LocalContentCache(fs, layout.cache.analysis.metrics, options.clock, options.random, {
|
|
84
|
+
maxBytes: analysisMaxBytes,
|
|
85
|
+
}),
|
|
86
|
+
);
|
|
87
|
+
const checkpoints = new LocalCheckpointStore(fs, layout.jobs, options.random);
|
|
88
|
+
const locks = new FileLockProvider(fs, layout.locks, options.clock, options.random, options.host);
|
|
89
|
+
const searchJourneys = new LocalSearchJourneyStore(
|
|
90
|
+
fs,
|
|
91
|
+
layout.productComplexity.sessions,
|
|
92
|
+
options.random,
|
|
93
|
+
locks,
|
|
94
|
+
options.clock,
|
|
95
|
+
);
|
|
96
|
+
|
|
97
|
+
return { layout, fs, embeddingCache, analysisCache, checkpoints, locks, searchJourneys };
|
|
98
|
+
}
|