@serve.zone/gitops 32.1.0 → 32.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.md +18 -0
- package/deno.json +1 -1
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes/actionlog.d.ts +1 -1
- package/dist_ts/classes/actionlog.js +1 -1
- package/dist_ts/classes/connectionmanager.d.ts +1 -5
- package/dist_ts/classes/connectionmanager.js +5 -28
- package/dist_ts/classes/gitopsapp.d.ts +8 -3
- package/dist_ts/classes/gitopsapp.js +29 -18
- package/dist_ts/{cache/classes.secrets.scan.service.d.ts → classes/secrets.scan.service.d.ts} +1 -1
- package/dist_ts/{cache/classes.secrets.scan.service.js → classes/secrets.scan.service.js} +3 -3
- package/dist_ts/{cache → db}/classes.cache.cleaner.d.ts +1 -3
- package/dist_ts/db/classes.cache.cleaner.js +59 -0
- package/dist_ts/{cache → db}/classes.cached.document.d.ts +1 -1
- package/dist_ts/{cache → db}/classes.cached.document.js +2 -2
- package/dist_ts/db/classes.gitops-db.d.ts +34 -0
- package/dist_ts/db/classes.gitops-db.js +77 -0
- package/dist_ts/{cache → db}/documents/classes.cached.project.js +3 -3
- package/dist_ts/{cache → db}/documents/classes.cached.secret.js +3 -3
- package/dist_ts/db/documents/classes.storage.record.d.ts +14 -0
- package/dist_ts/db/documents/classes.storage.record.js +82 -0
- package/dist_ts/{cache → db}/documents/index.d.ts +1 -0
- package/dist_ts/db/documents/index.js +4 -0
- package/dist_ts/db/index.d.ts +5 -0
- package/dist_ts/db/index.js +5 -0
- package/dist_ts/paths.d.ts +2 -2
- package/dist_ts/paths.js +2 -3
- package/dist_ts/plugins.d.ts +2 -3
- package/dist_ts/plugins.js +6 -4
- package/dist_ts/storage/classes.storagemanager.d.ts +7 -16
- package/dist_ts/storage/classes.storagemanager.js +54 -94
- package/dist_ts/storage/index.d.ts +0 -1
- package/dist_ts_migrations/classes.migration-record.d.ts +11 -0
- package/dist_ts_migrations/classes.migration-record.js +80 -0
- package/dist_ts_migrations/import-filesystem-records.d.ts +9 -0
- package/dist_ts_migrations/import-filesystem-records.js +74 -0
- package/dist_ts_migrations/index.d.ts +12 -0
- package/dist_ts_migrations/index.js +24 -0
- package/dist_ts_migrations/interfaces.d.ts +16 -0
- package/dist_ts_migrations/interfaces.js +2 -0
- package/package.json +6 -4
- package/readme.md +21 -14
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes/actionlog.ts +1 -1
- package/ts/classes/connectionmanager.ts +4 -27
- package/ts/classes/gitopsapp.ts +36 -17
- package/ts/{cache/classes.secrets.scan.service.ts → classes/secrets.scan.service.ts} +3 -3
- package/ts/{cache → db}/classes.cache.cleaner.ts +1 -4
- package/ts/{cache → db}/classes.cached.document.ts +1 -1
- package/ts/db/classes.gitops-db.ts +92 -0
- package/ts/{cache → db}/documents/classes.cached.project.ts +2 -2
- package/ts/{cache → db}/documents/classes.cached.secret.ts +2 -2
- package/ts/db/documents/classes.storage.record.ts +26 -0
- package/ts/{cache → db}/documents/index.ts +1 -0
- package/ts/db/index.ts +5 -0
- package/ts/paths.ts +3 -4
- package/ts/plugins.ts +5 -3
- package/ts/storage/classes.storagemanager.ts +59 -95
- package/ts/storage/index.ts +0 -1
- package/ts_migrations/classes.migration-record.ts +23 -0
- package/ts_migrations/import-filesystem-records.ts +79 -0
- package/ts_migrations/index.ts +31 -0
- package/ts_migrations/interfaces.ts +18 -0
- package/ts_web/00_commitinfo_data.ts +1 -1
- package/dist_ts/cache/classes.cache.cleaner.js +0 -61
- package/dist_ts/cache/classes.cachedb.d.ts +0 -22
- package/dist_ts/cache/classes.cachedb.js +0 -58
- package/dist_ts/cache/documents/index.js +0 -3
- package/dist_ts/cache/index.d.ts +0 -7
- package/dist_ts/cache/index.js +0 -6
- package/ts/cache/classes.cachedb.ts +0 -72
- package/ts/cache/index.ts +0 -7
- /package/dist_ts/{cache → db}/documents/classes.cached.project.d.ts +0 -0
- /package/dist_ts/{cache → db}/documents/classes.cached.secret.d.ts +0 -0
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import * as plugins from '../plugins.js';
|
|
2
|
+
import { logger } from '../logging.js';
|
|
3
|
+
|
|
4
|
+
export interface IGitopsDbOptions {
|
|
5
|
+
/** Connection URL of the document database gitops persists into. */
|
|
6
|
+
mongoDbUrl: string;
|
|
7
|
+
/** Database name inside that server. */
|
|
8
|
+
mongoDbName: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
const requireEnv = (keyArg: string): string => {
|
|
12
|
+
const value = process.env[keyArg];
|
|
13
|
+
if (!value) {
|
|
14
|
+
throw new Error(`GitOps requires ${keyArg} to be set and non-empty.`);
|
|
15
|
+
}
|
|
16
|
+
if (value.trim() !== value) {
|
|
17
|
+
throw new Error(`GitOps requires ${keyArg} to be canonical without surrounding whitespace.`);
|
|
18
|
+
}
|
|
19
|
+
return value;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Reads the database binding from the process environment. There is no default and no
|
|
24
|
+
* embedded fallback: gitops keeps every durable record — connections, jobs, sync
|
|
25
|
+
* configuration, managed secrets, the action log and the provider caches — in this one
|
|
26
|
+
* database, so a missing binding has to stop startup rather than silently produce an
|
|
27
|
+
* instance that forgets everything it was told.
|
|
28
|
+
*/
|
|
29
|
+
export const readGitopsDbOptionsFromEnvironment = (): IGitopsDbOptions => ({
|
|
30
|
+
mongoDbUrl: requireEnv('GITOPS_MONGODB_URL'),
|
|
31
|
+
// An explicitly set but empty name is a misconfiguration, not a request for the default.
|
|
32
|
+
mongoDbName: process.env.GITOPS_MONGODB_NAME === undefined ? 'gitops' : requireEnv('GITOPS_MONGODB_NAME'),
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* The single document database behind gitops.
|
|
37
|
+
*
|
|
38
|
+
* Every model binds its collection through `GitopsDb.getInstance().getDb()`, so the
|
|
39
|
+
* singleton is the only place that knows the connection. Tests construct it with an
|
|
40
|
+
* explicit binding to an isolated server; the application reads the binding from the
|
|
41
|
+
* environment.
|
|
42
|
+
*/
|
|
43
|
+
export class GitopsDb {
|
|
44
|
+
private static instance: GitopsDb | null = null;
|
|
45
|
+
|
|
46
|
+
private smartdataDb: plugins.smartdata.SmartdataDb | null = null;
|
|
47
|
+
private readonly options: IGitopsDbOptions;
|
|
48
|
+
|
|
49
|
+
private constructor(options: IGitopsDbOptions) {
|
|
50
|
+
this.options = options;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
static getInstance(options?: IGitopsDbOptions): GitopsDb {
|
|
54
|
+
if (!GitopsDb.instance) {
|
|
55
|
+
if (!options) {
|
|
56
|
+
throw new Error('GitopsDb has no instance yet — the first call must supply its options.');
|
|
57
|
+
}
|
|
58
|
+
GitopsDb.instance = new GitopsDb(options);
|
|
59
|
+
}
|
|
60
|
+
return GitopsDb.instance;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
static resetInstance(): void {
|
|
64
|
+
GitopsDb.instance = null;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
async start(): Promise<void> {
|
|
68
|
+
logger.info('Connecting to the gitops database...');
|
|
69
|
+
const smartdataDb = new plugins.smartdata.SmartdataDb({
|
|
70
|
+
mongoDbUrl: this.options.mongoDbUrl,
|
|
71
|
+
mongoDbName: this.options.mongoDbName,
|
|
72
|
+
});
|
|
73
|
+
await smartdataDb.init();
|
|
74
|
+
this.smartdataDb = smartdataDb;
|
|
75
|
+
logger.success(`Connected to the gitops database (db: ${this.options.mongoDbName})`);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
async stop(): Promise<void> {
|
|
79
|
+
if (!this.smartdataDb) return;
|
|
80
|
+
const smartdataDb = this.smartdataDb;
|
|
81
|
+
this.smartdataDb = null;
|
|
82
|
+
await smartdataDb.close();
|
|
83
|
+
logger.success('Disconnected from the gitops database');
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
getDb(): plugins.smartdata.SmartdataDb {
|
|
87
|
+
if (!this.smartdataDb) {
|
|
88
|
+
throw new Error('GitopsDb not started. Call start() first.');
|
|
89
|
+
}
|
|
90
|
+
return this.smartdataDb;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import * as plugins from '../../plugins.js';
|
|
2
|
-
import {
|
|
2
|
+
import { GitopsDb } from '../classes.gitops-db.js';
|
|
3
3
|
import { CachedDocument, TTL } from '../classes.cached.document.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Cached project data from git providers. TTL: 5 minutes.
|
|
7
7
|
*/
|
|
8
|
-
@plugins.smartdata.Collection(() =>
|
|
8
|
+
@plugins.smartdata.Collection(() => GitopsDb.getInstance().getDb())
|
|
9
9
|
export class CachedProject extends CachedDocument<CachedProject> {
|
|
10
10
|
@plugins.smartdata.unI()
|
|
11
11
|
public id: string = '';
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import * as plugins from '../../plugins.js';
|
|
2
|
-
import {
|
|
2
|
+
import { GitopsDb } from '../classes.gitops-db.js';
|
|
3
3
|
import { CachedDocument, TTL } from '../classes.cached.document.js';
|
|
4
4
|
import type { ISecret } from '../../../ts_interfaces/data/secret.js';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* Cached secret data from git providers. TTL: 24 hours.
|
|
8
8
|
*/
|
|
9
|
-
@plugins.smartdata.Collection(() =>
|
|
9
|
+
@plugins.smartdata.Collection(() => GitopsDb.getInstance().getDb())
|
|
10
10
|
export class CachedSecret extends CachedDocument<CachedSecret> {
|
|
11
11
|
@plugins.smartdata.unI()
|
|
12
12
|
public id: string = '';
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import * as plugins from '../../plugins.js';
|
|
2
|
+
import { GitopsDb } from '../classes.gitops-db.js';
|
|
3
|
+
|
|
4
|
+
const getDb = () => GitopsDb.getInstance().getDb();
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* One durable key/value record.
|
|
8
|
+
*
|
|
9
|
+
* `key` owns the stored `_id`, so the primary key is the single uniqueness authority: a
|
|
10
|
+
* write is one identity-anchored upsert with no second index to disagree with, and a
|
|
11
|
+
* prefix listing is answered by the `_id` index instead of a collection scan. That is what
|
|
12
|
+
* `identityAsDocumentId: 'key'` declares, and it holds for every record this collection
|
|
13
|
+
* ever received, because the store below has always derived `_id` from the key.
|
|
14
|
+
*/
|
|
15
|
+
@plugins.smartdata.Collection(() => getDb(), { identityAsDocumentId: 'key' })
|
|
16
|
+
export class StorageRecord extends plugins.smartdata.SmartDataDbDoc<
|
|
17
|
+
StorageRecord,
|
|
18
|
+
StorageRecord
|
|
19
|
+
> {
|
|
20
|
+
@plugins.smartdata.unI()
|
|
21
|
+
@plugins.smartdata.svDb()
|
|
22
|
+
public key!: string;
|
|
23
|
+
|
|
24
|
+
@plugins.smartdata.svDb()
|
|
25
|
+
public value!: string;
|
|
26
|
+
}
|
package/ts/db/index.ts
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export { GitopsDb, readGitopsDbOptionsFromEnvironment } from './classes.gitops-db.js';
|
|
2
|
+
export type { IGitopsDbOptions } from './classes.gitops-db.js';
|
|
3
|
+
export { CachedDocument, TTL } from './classes.cached.document.js';
|
|
4
|
+
export { CacheCleaner } from './classes.cache.cleaner.js';
|
|
5
|
+
export * from './documents/index.js';
|
package/ts/paths.ts
CHANGED
|
@@ -2,8 +2,8 @@ import * as plugins from './plugins.js';
|
|
|
2
2
|
|
|
3
3
|
export interface IGitopsPaths {
|
|
4
4
|
gitopsHomeDir: string;
|
|
5
|
-
|
|
6
|
-
|
|
5
|
+
/** Directory the releases before the document database wrote key/value records into. */
|
|
6
|
+
legacyStoragePath: string;
|
|
7
7
|
}
|
|
8
8
|
|
|
9
9
|
/**
|
|
@@ -13,7 +13,6 @@ export function resolvePaths(baseDir?: string): IGitopsPaths {
|
|
|
13
13
|
const home = baseDir ?? plugins.path.join(process.env.HOME ?? '/tmp', '.serve.zone', 'gitops');
|
|
14
14
|
return {
|
|
15
15
|
gitopsHomeDir: home,
|
|
16
|
-
|
|
17
|
-
defaultTsmDbPath: plugins.path.join(home, 'tsmdb'),
|
|
16
|
+
legacyStoragePath: plugins.path.join(home, 'storage'),
|
|
18
17
|
};
|
|
19
18
|
}
|
package/ts/plugins.ts
CHANGED
|
@@ -30,9 +30,11 @@ import * as bookstackClient from '@apiclient.xyz/bookstack';
|
|
|
30
30
|
export { giteaClient, gitlabClient, bookstackClient };
|
|
31
31
|
|
|
32
32
|
// Database
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
33
|
+
// The document ORM. Exported as `smartdata` because that is the name every call site uses and
|
|
34
|
+
// because `@lossless.org/client/nosqldb` supersedes `@push.rocks/smartdata` with an identical
|
|
35
|
+
// API surface; the persisted shape is unchanged.
|
|
36
|
+
import * as smartdata from '@lossless.org/client/nosqldb';
|
|
37
|
+
export { smartdata };
|
|
36
38
|
|
|
37
39
|
// Secrets
|
|
38
40
|
import * as smartsecret from '@push.rocks/smartsecret';
|
|
@@ -1,31 +1,27 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { StorageRecord } from '../db/documents/classes.storage.record.js';
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
3
|
+
/**
|
|
4
|
+
* The upper bound of the key range scanned for a prefix listing. `` is above every
|
|
5
|
+
* character a normalized key may contain, so `[prefix/, prefix/)` covers the whole
|
|
6
|
+
* subtree of a prefix and nothing beyond it.
|
|
7
|
+
*/
|
|
8
|
+
const KEY_RANGE_END = '';
|
|
9
9
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
10
|
+
/**
|
|
11
|
+
* The key is the record's primary key, and MongoDB indexes a primary key of at most
|
|
12
|
+
* 1024 bytes; the bound is stated here so an oversized key is refused by name instead of
|
|
13
|
+
* surfacing as a driver index error on write.
|
|
14
|
+
*/
|
|
15
|
+
const MAX_KEY_BYTES = 1024;
|
|
13
16
|
|
|
14
17
|
/**
|
|
15
|
-
* Key-value storage
|
|
16
|
-
*
|
|
18
|
+
* Key-value storage for gitops' durable records, kept in the `StorageRecord` collection.
|
|
19
|
+
*
|
|
20
|
+
* Keys must start with '/' and are normalized (no '..', no double slashes). Listing is
|
|
21
|
+
* non-recursive: it returns the direct children of a prefix, which is what every caller
|
|
22
|
+
* relies on to keep neighbouring key spaces apart.
|
|
17
23
|
*/
|
|
18
24
|
export class StorageManager {
|
|
19
|
-
private backend: TStorageBackend;
|
|
20
|
-
private fsPath: string;
|
|
21
|
-
private memoryStore: Map<string, string>;
|
|
22
|
-
|
|
23
|
-
constructor(config: IStorageConfig = {}) {
|
|
24
|
-
this.backend = config.backend ?? 'filesystem';
|
|
25
|
-
this.fsPath = config.fsPath ?? './storage';
|
|
26
|
-
this.memoryStore = new Map();
|
|
27
|
-
}
|
|
28
|
-
|
|
29
25
|
/**
|
|
30
26
|
* Normalize and validate a storage key.
|
|
31
27
|
*/
|
|
@@ -35,101 +31,69 @@ export class StorageManager {
|
|
|
35
31
|
}
|
|
36
32
|
// Strip '..' segments and normalize double slashes
|
|
37
33
|
const segments = key.split('/').filter((s) => s !== '' && s !== '..');
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
*/
|
|
44
|
-
private keyToPath(key: string): string {
|
|
45
|
-
const normalized = this.normalizeKey(key);
|
|
46
|
-
return plugins.path.join(this.fsPath, ...normalized.split('/').filter(Boolean));
|
|
34
|
+
const normalized = '/' + segments.join('/');
|
|
35
|
+
if (Buffer.byteLength(normalized, 'utf8') > MAX_KEY_BYTES) {
|
|
36
|
+
throw new Error(`Storage key exceeds ${MAX_KEY_BYTES} bytes: ${normalized.slice(0, 64)}…`);
|
|
37
|
+
}
|
|
38
|
+
return normalized;
|
|
47
39
|
}
|
|
48
40
|
|
|
49
41
|
async get(key: string): Promise<string | null> {
|
|
50
|
-
const
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
try {
|
|
55
|
-
return await plugins.fs.readFile(this.keyToPath(normalized), 'utf8');
|
|
56
|
-
} catch (err) {
|
|
57
|
-
if (isNotFoundError(err)) return null;
|
|
58
|
-
throw err;
|
|
59
|
-
}
|
|
42
|
+
const record = await StorageRecord.getInstance<StorageRecord>({
|
|
43
|
+
key: this.normalizeKey(key),
|
|
44
|
+
});
|
|
45
|
+
return record ? record.value : null;
|
|
60
46
|
}
|
|
61
47
|
|
|
62
48
|
async set(key: string, value: string): Promise<void> {
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
// Atomic write: write to temp then rename
|
|
72
|
-
const tmpPath = filePath + '.tmp';
|
|
73
|
-
await plugins.fs.writeFile(tmpPath, value, 'utf8');
|
|
74
|
-
await plugins.fs.rename(tmpPath, filePath);
|
|
49
|
+
// The equality anchor on `key` is what makes this a single race-free upsert: the
|
|
50
|
+
// primary key is derived from the filter, and the client seeds the immutable identity
|
|
51
|
+
// from that same anchor, so the key is never restated in the update document.
|
|
52
|
+
await StorageRecord.atomicUpdate<StorageRecord>(
|
|
53
|
+
{ key: this.normalizeKey(key) },
|
|
54
|
+
{ $set: { value } },
|
|
55
|
+
{ upsert: true },
|
|
56
|
+
);
|
|
75
57
|
}
|
|
76
58
|
|
|
77
59
|
async delete(key: string): Promise<boolean> {
|
|
78
60
|
const normalized = this.normalizeKey(key);
|
|
79
|
-
if (
|
|
80
|
-
return
|
|
81
|
-
}
|
|
82
|
-
try {
|
|
83
|
-
await plugins.fs.rm(this.keyToPath(normalized));
|
|
84
|
-
return true;
|
|
85
|
-
} catch (err) {
|
|
86
|
-
if (isNotFoundError(err)) return false;
|
|
87
|
-
throw err;
|
|
61
|
+
if (!(await StorageRecord.exists<StorageRecord>({ key: normalized }))) {
|
|
62
|
+
return false;
|
|
88
63
|
}
|
|
64
|
+
// The identity pins a single document, which is what a singular delete requires.
|
|
65
|
+
await StorageRecord.atomicDelete<StorageRecord>({ key: normalized });
|
|
66
|
+
return true;
|
|
89
67
|
}
|
|
90
68
|
|
|
91
69
|
async exists(key: string): Promise<boolean> {
|
|
92
|
-
|
|
93
|
-
if (this.backend === 'memory') {
|
|
94
|
-
return this.memoryStore.has(normalized);
|
|
95
|
-
}
|
|
96
|
-
try {
|
|
97
|
-
await plugins.fs.stat(this.keyToPath(normalized));
|
|
98
|
-
return true;
|
|
99
|
-
} catch (err) {
|
|
100
|
-
if (isNotFoundError(err)) return false;
|
|
101
|
-
throw err;
|
|
102
|
-
}
|
|
70
|
+
return StorageRecord.exists<StorageRecord>({ key: this.normalizeKey(key) });
|
|
103
71
|
}
|
|
104
72
|
|
|
105
73
|
/**
|
|
106
|
-
* List keys
|
|
74
|
+
* List the keys directly below a given prefix. Records deeper in the subtree are read by
|
|
75
|
+
* the range scan and then dropped, mirroring the directory listing this replaced.
|
|
107
76
|
*/
|
|
108
77
|
async list(prefix: string): Promise<string[]> {
|
|
109
78
|
const normalized = this.normalizeKey(prefix);
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
const dirPath = this.keyToPath(normalized);
|
|
120
|
-
const keys: string[] = [];
|
|
79
|
+
// Normalization leaves the root as '/', which is already its own child prefix.
|
|
80
|
+
const childPrefix = normalized.endsWith('/') ? normalized : `${normalized}/`;
|
|
81
|
+
const cursor = await StorageRecord.getCursor<StorageRecord>(
|
|
82
|
+
{ key: { $gte: childPrefix, $lt: `${childPrefix}${KEY_RANGE_END}` } },
|
|
83
|
+
// Only the keys are wanted here; leaving the values in the database keeps a listing
|
|
84
|
+
// independent of how much each record holds.
|
|
85
|
+
{ projection: { key: 1 } },
|
|
86
|
+
);
|
|
87
|
+
let records: StorageRecord[];
|
|
121
88
|
try {
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
keys.push(normalized.replace(/\/$/, '') + '/' + entry.name);
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
} catch (err) {
|
|
129
|
-
if (isNotFoundError(err)) return [];
|
|
130
|
-
throw err;
|
|
89
|
+
records = await cursor.toArray();
|
|
90
|
+
} finally {
|
|
91
|
+
await cursor.close();
|
|
131
92
|
}
|
|
132
|
-
return
|
|
93
|
+
return records
|
|
94
|
+
.map((recordArg) => recordArg.key)
|
|
95
|
+
.filter((keyArg) => !keyArg.slice(childPrefix.length).includes('/'))
|
|
96
|
+
.sort();
|
|
133
97
|
}
|
|
134
98
|
|
|
135
99
|
async getJSON<T>(key: string): Promise<T | null> {
|
package/ts/storage/index.ts
CHANGED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import * as plugins from '../ts/plugins.js';
|
|
2
|
+
import { GitopsDb } from '../ts/db/index.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* The ledger of migrations that have run to completion.
|
|
6
|
+
*
|
|
7
|
+
* `migration` owns the stored `_id`, so a migration is looked up by primary key and can
|
|
8
|
+
* only ever be recorded once.
|
|
9
|
+
*/
|
|
10
|
+
@plugins.smartdata.Collection(() => GitopsDb.getInstance().getDb(), {
|
|
11
|
+
identityAsDocumentId: 'migration',
|
|
12
|
+
})
|
|
13
|
+
export class MigrationRecord extends plugins.smartdata.SmartDataDbDoc<
|
|
14
|
+
MigrationRecord,
|
|
15
|
+
MigrationRecord
|
|
16
|
+
> {
|
|
17
|
+
@plugins.smartdata.unI()
|
|
18
|
+
@plugins.smartdata.svDb()
|
|
19
|
+
public migration!: string;
|
|
20
|
+
|
|
21
|
+
@plugins.smartdata.svDb()
|
|
22
|
+
public completedAt!: number;
|
|
23
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import * as plugins from '../ts/plugins.js';
|
|
2
|
+
import type { IMigration, IMigrationContext } from './interfaces.js';
|
|
3
|
+
|
|
4
|
+
interface ILegacyConnection {
|
|
5
|
+
id: string;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Reads every regular file below `directoryArg` and yields it as a storage key relative to
|
|
10
|
+
* the tree root, so `<root>/connections/a.json` becomes `/connections/a.json`.
|
|
11
|
+
*/
|
|
12
|
+
const readDirectoryEntries = async (directoryArg: string) => {
|
|
13
|
+
try {
|
|
14
|
+
return await plugins.fs.readdir(directoryArg, { withFileTypes: true });
|
|
15
|
+
} catch (error) {
|
|
16
|
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return [];
|
|
17
|
+
throw error;
|
|
18
|
+
}
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
const readLegacyTree = async (
|
|
22
|
+
directoryArg: string,
|
|
23
|
+
prefixArg: string,
|
|
24
|
+
): Promise<{ key: string; value: string }[]> => {
|
|
25
|
+
const entries = await readDirectoryEntries(directoryArg);
|
|
26
|
+
const records: { key: string; value: string }[] = [];
|
|
27
|
+
for (const entry of entries) {
|
|
28
|
+
const entryPath = plugins.path.join(directoryArg, entry.name);
|
|
29
|
+
if (entry.isDirectory()) {
|
|
30
|
+
records.push(...(await readLegacyTree(entryPath, `${prefixArg}/${entry.name}`)));
|
|
31
|
+
} else if (entry.isFile()) {
|
|
32
|
+
records.push({
|
|
33
|
+
key: `${prefixArg}/${entry.name}`,
|
|
34
|
+
value: await plugins.fs.readFile(entryPath, 'utf8'),
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return records;
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Imports the key/value records that earlier releases kept as files.
|
|
43
|
+
*
|
|
44
|
+
* Both sources are read once and never written back: the ledger, not a renamed file, is
|
|
45
|
+
* what records that this ran. The legacy tree is left on disk untouched so an operator can
|
|
46
|
+
* archive it after confirming the import.
|
|
47
|
+
*/
|
|
48
|
+
export const importFilesystemRecords: IMigration = {
|
|
49
|
+
name: 'import-filesystem-records',
|
|
50
|
+
run: async (contextArg: IMigrationContext): Promise<void> => {
|
|
51
|
+
// The development-time connections file was the authority over the tree before, and
|
|
52
|
+
// stays it here: it is written first and the tree import below skips existing keys.
|
|
53
|
+
try {
|
|
54
|
+
const legacyText = await plugins.fs.readFile(contextArg.legacyConnectionsFile, 'utf8');
|
|
55
|
+
const legacyConnections = JSON.parse(legacyText) as ILegacyConnection[];
|
|
56
|
+
for (const connection of legacyConnections) {
|
|
57
|
+
await contextArg.storage.setJSON(`/connections/${connection.id}.json`, connection);
|
|
58
|
+
}
|
|
59
|
+
if (legacyConnections.length > 0) {
|
|
60
|
+
contextArg.log(`imported ${legacyConnections.length} connection(s) from the legacy file`);
|
|
61
|
+
}
|
|
62
|
+
} catch (error) {
|
|
63
|
+
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const records = await readLegacyTree(contextArg.legacyStoragePath, '');
|
|
67
|
+
let imported = 0;
|
|
68
|
+
for (const record of records) {
|
|
69
|
+
// A key that already exists was written by a newer authority — the legacy file above
|
|
70
|
+
// or a release that already ran against the database — so it is never overwritten.
|
|
71
|
+
if (await contextArg.storage.exists(record.key)) continue;
|
|
72
|
+
await contextArg.storage.set(record.key, record.value);
|
|
73
|
+
imported++;
|
|
74
|
+
}
|
|
75
|
+
if (imported > 0) {
|
|
76
|
+
contextArg.log(`imported ${imported} record(s) from ${contextArg.legacyStoragePath}`);
|
|
77
|
+
}
|
|
78
|
+
},
|
|
79
|
+
};
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { MigrationRecord } from './classes.migration-record.js';
|
|
2
|
+
import { importFilesystemRecords } from './import-filesystem-records.js';
|
|
3
|
+
import type { IMigration, IMigrationContext } from './interfaces.js';
|
|
4
|
+
|
|
5
|
+
export type { IMigration, IMigrationContext } from './interfaces.js';
|
|
6
|
+
export { MigrationRecord } from './classes.migration-record.js';
|
|
7
|
+
|
|
8
|
+
/** Applied in order; a name that is already in the ledger is skipped. */
|
|
9
|
+
const migrations: readonly IMigration[] = [importFilesystemRecords];
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Runs the pending migrations against an already connected database.
|
|
13
|
+
*
|
|
14
|
+
* Called from startup only, after the database is up and before any manager reads a
|
|
15
|
+
* record. No lease is taken: every migration is idempotent, so two instances starting at
|
|
16
|
+
* the same time converge on the same result, while a lease would let one crashed claimant
|
|
17
|
+
* block every later start.
|
|
18
|
+
*/
|
|
19
|
+
export const runStartupMigrations = async (contextArg: IMigrationContext): Promise<void> => {
|
|
20
|
+
for (const migration of migrations) {
|
|
21
|
+
if (await MigrationRecord.exists<MigrationRecord>({ migration: migration.name })) continue;
|
|
22
|
+
contextArg.log(`running migration ${migration.name}`);
|
|
23
|
+
await migration.run(contextArg);
|
|
24
|
+
await MigrationRecord.atomicUpdate<MigrationRecord>(
|
|
25
|
+
{ migration: migration.name },
|
|
26
|
+
{ $set: { completedAt: Date.now() } },
|
|
27
|
+
{ upsert: true },
|
|
28
|
+
);
|
|
29
|
+
contextArg.log(`migration ${migration.name} completed`);
|
|
30
|
+
}
|
|
31
|
+
};
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { StorageManager } from '../ts/storage/index.js';
|
|
2
|
+
|
|
3
|
+
export interface IMigrationContext {
|
|
4
|
+
/** The document-backed key/value store every migration writes through. */
|
|
5
|
+
storage: StorageManager;
|
|
6
|
+
/** Root of the key/value tree the releases before the document database wrote. */
|
|
7
|
+
legacyStoragePath: string;
|
|
8
|
+
/** The development-time connection list earlier releases imported on every start. */
|
|
9
|
+
legacyConnectionsFile: string;
|
|
10
|
+
/** Reports progress to the application log. */
|
|
11
|
+
log: (messageArg: string) => void;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export interface IMigration {
|
|
15
|
+
/** Stable identity recorded in the ledger once the migration completed. */
|
|
16
|
+
name: string;
|
|
17
|
+
run: (contextArg: IMigrationContext) => Promise<void>;
|
|
18
|
+
}
|
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
import { logger } from '../logging.js';
|
|
2
|
-
import { unrefTimer } from '../timers.js';
|
|
3
|
-
const DEFAULT_INTERVAL_MS = 60 * 60 * 1000; // 1 hour
|
|
4
|
-
/**
|
|
5
|
-
* Periodically cleans up expired cached documents.
|
|
6
|
-
*/
|
|
7
|
-
export class CacheCleaner {
|
|
8
|
-
intervalId = null;
|
|
9
|
-
intervalMs;
|
|
10
|
-
documentClasses = [];
|
|
11
|
-
cacheDb;
|
|
12
|
-
constructor(cacheDb, intervalMs = DEFAULT_INTERVAL_MS) {
|
|
13
|
-
this.cacheDb = cacheDb;
|
|
14
|
-
this.intervalMs = intervalMs;
|
|
15
|
-
}
|
|
16
|
-
/** Register a document class for cleanup */
|
|
17
|
-
registerClass(cls) {
|
|
18
|
-
this.documentClasses.push(cls);
|
|
19
|
-
}
|
|
20
|
-
start() {
|
|
21
|
-
if (this.intervalId !== null)
|
|
22
|
-
return;
|
|
23
|
-
this.intervalId = setInterval(() => {
|
|
24
|
-
this.clean().catch((err) => {
|
|
25
|
-
logger.error(`CacheCleaner error: ${err}`);
|
|
26
|
-
});
|
|
27
|
-
}, this.intervalMs);
|
|
28
|
-
// Unref so the interval doesn't prevent process exit
|
|
29
|
-
unrefTimer(this.intervalId);
|
|
30
|
-
logger.debug(`CacheCleaner started (interval: ${this.intervalMs}ms)`);
|
|
31
|
-
}
|
|
32
|
-
stop() {
|
|
33
|
-
if (this.intervalId !== null) {
|
|
34
|
-
clearInterval(this.intervalId);
|
|
35
|
-
this.intervalId = null;
|
|
36
|
-
logger.debug('CacheCleaner stopped');
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
/** Run a single cleanup pass */
|
|
40
|
-
async clean() {
|
|
41
|
-
const now = Date.now();
|
|
42
|
-
let totalDeleted = 0;
|
|
43
|
-
for (const cls of this.documentClasses) {
|
|
44
|
-
try {
|
|
45
|
-
const expired = await cls.getInstances({ expiresAt: { $lt: now } });
|
|
46
|
-
for (const doc of expired) {
|
|
47
|
-
await doc.delete();
|
|
48
|
-
totalDeleted++;
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
catch (err) {
|
|
52
|
-
logger.error(`CacheCleaner: failed to clean class: ${err}`);
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
if (totalDeleted > 0) {
|
|
56
|
-
logger.debug(`CacheCleaner: deleted ${totalDeleted} expired document(s)`);
|
|
57
|
-
}
|
|
58
|
-
return totalDeleted;
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY2xhc3Nlcy5jYWNoZS5jbGVhbmVyLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vdHMvY2FjaGUvY2xhc3Nlcy5jYWNoZS5jbGVhbmVyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxNQUFNLEVBQUUsTUFBTSxlQUFlLENBQUM7QUFDdkMsT0FBTyxFQUFFLFVBQVUsRUFBRSxNQUFNLGNBQWMsQ0FBQztBQU0xQyxNQUFNLG1CQUFtQixHQUFHLEVBQUUsR0FBRyxFQUFFLEdBQUcsSUFBSSxDQUFDLENBQUMsU0FBUztBQUVyRDs7R0FFRztBQUNILE1BQU0sT0FBTyxZQUFZO0lBQ2YsVUFBVSxHQUEwQyxJQUFJLENBQUM7SUFDekQsVUFBVSxDQUFTO0lBQ25CLGVBQWUsR0FBb0IsRUFBRSxDQUFDO0lBQ3RDLE9BQU8sQ0FBVTtJQUV6QixZQUFZLE9BQWdCLEVBQUUsVUFBVSxHQUFHLG1CQUFtQjtRQUM1RCxJQUFJLENBQUMsT0FBTyxHQUFHLE9BQU8sQ0FBQztRQUN2QixJQUFJLENBQUMsVUFBVSxHQUFHLFVBQVUsQ0FBQztJQUMvQixDQUFDO0lBRUQsNENBQTRDO0lBQzVDLGFBQWEsQ0FBQyxHQUFrQjtRQUM5QixJQUFJLENBQUMsZUFBZSxDQUFDLElBQUksQ0FBQyxHQUFHLENBQUMsQ0FBQztJQUNqQyxDQUFDO0lBRUQsS0FBSztRQUNILElBQUksSUFBSSxDQUFDLFVBQVUsS0FBSyxJQUFJO1lBQUUsT0FBTztRQUNyQyxJQUFJLENBQUMsVUFBVSxHQUFHLFdBQVcsQ0FBQyxHQUFHLEVBQUU7WUFDakMsSUFBSSxDQUFDLEtBQUssRUFBRSxDQUFDLEtBQUssQ0FBQyxDQUFDLEdBQUcsRUFBRSxFQUFFO2dCQUN6QixNQUFNLENBQUMsS0FBSyxDQUFDLHVCQUF1QixHQUFHLEVBQUUsQ0FBQyxDQUFDO1lBQzdDLENBQUMsQ0FBQyxDQUFDO1FBQ0wsQ0FBQyxFQUFFLElBQUksQ0FBQyxVQUFVLENBQUMsQ0FBQztRQUNwQixxREFBcUQ7UUFDckQsVUFBVSxDQUFDLElBQUksQ0FBQyxVQUFVLENBQUMsQ0FBQztRQUM1QixNQUFNLENBQUMsS0FBSyxDQUFDLG1DQUFtQyxJQUFJLENBQUMsVUFBVSxLQUFLLENBQUMsQ0FBQztJQUN4RSxDQUFDO0lBRUQsSUFBSTtRQUNGLElBQUksSUFBSSxDQUFDLFVBQVUsS0FBSyxJQUFJLEVBQUUsQ0FBQztZQUM3QixhQUFhLENBQUMsSUFBSSxDQUFDLFVBQVUsQ0FBQyxDQUFDO1lBQy9CLElBQUksQ0FBQyxVQUFVLEdBQUcsSUFBSSxDQUFDO1lBQ3ZCLE1BQU0sQ0FBQyxLQUFLLENBQUMsc0JBQXNCLENBQUMsQ0FBQztRQUN2QyxDQUFDO0lBQ0gsQ0FBQztJQUVELGdDQUFnQztJQUNoQyxLQUFLLENBQUMsS0FBSztRQUNULE1BQU0sR0FBRyxHQUFHLElBQUksQ0FBQyxHQUFHLEVBQUUsQ0FBQztRQUN2QixJQUFJLFlBQVksR0FBRyxDQUFDLENBQUM7UUFDckIsS0FBSyxNQUFNLEdBQUcsSUFBSSxJQUFJLENBQUMsZUFBZSxFQUFFLENBQUM7WUFDdkMsSUFBSSxDQUFDO2dCQUNILE1BQU0sT0FBTyxHQUFHLE1BQU0sR0FBRyxDQUFDLFlBQVksQ0FBQyxFQUFFLFNBQVMsRUFBRSxFQUFFLEdBQUcsRUFBRSxHQUFHLEVBQUUsRUFBRSxDQUFDLENBQUM7Z0JBQ3BFLEtBQUssTUFBTSxHQUFHLElBQUksT0FBTyxFQUFFLENBQUM7b0JBQzFCLE1BQU0sR0FBRyxDQUFDLE1BQU0sRUFBRSxDQUFDO29CQUNuQixZQUFZLEVBQUUsQ0FBQztnQkFDakIsQ0FBQztZQUNILENBQUM7WUFBQyxPQUFPLEdBQUcsRUFBRSxDQUFDO2dCQUNiLE1BQU0sQ0FBQyxLQUFLLENBQUMsd0NBQXdDLEdBQUcsRUFBRSxDQUFDLENBQUM7WUFDOUQsQ0FBQztRQUNILENBQUM7UUFDRCxJQUFJLFlBQVksR0FBRyxDQUFDLEVBQUUsQ0FBQztZQUNyQixNQUFNLENBQUMsS0FBSyxDQUFDLHlCQUF5QixZQUFZLHNCQUFzQixDQUFDLENBQUM7UUFDNUUsQ0FBQztRQUNELE9BQU8sWUFBWSxDQUFDO0lBQ3RCLENBQUM7Q0FDRiJ9
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import * as plugins from '../plugins.js';
|
|
2
|
-
export interface ICacheDbOptions {
|
|
3
|
-
storagePath?: string;
|
|
4
|
-
dbName?: string;
|
|
5
|
-
debug?: boolean;
|
|
6
|
-
}
|
|
7
|
-
/**
|
|
8
|
-
* Singleton wrapper around SmartMongo's NoSQLDB memory server and SmartData.
|
|
9
|
-
* Provides a managed MongoDB-compatible cache database.
|
|
10
|
-
*/
|
|
11
|
-
export declare class CacheDb {
|
|
12
|
-
private static instance;
|
|
13
|
-
private smartMongo;
|
|
14
|
-
private smartdataDb;
|
|
15
|
-
private options;
|
|
16
|
-
private constructor();
|
|
17
|
-
static getInstance(options?: ICacheDbOptions): CacheDb;
|
|
18
|
-
static resetInstance(): void;
|
|
19
|
-
start(): Promise<void>;
|
|
20
|
-
stop(): Promise<void>;
|
|
21
|
-
getDb(): InstanceType<typeof plugins.smartdata.SmartdataDb>;
|
|
22
|
-
}
|