@serve.zone/gitops 32.1.0 → 32.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/changelog.md +18 -0
  2. package/deno.json +1 -1
  3. package/dist_ts/00_commitinfo_data.js +1 -1
  4. package/dist_ts/classes/actionlog.d.ts +1 -1
  5. package/dist_ts/classes/actionlog.js +1 -1
  6. package/dist_ts/classes/connectionmanager.d.ts +1 -5
  7. package/dist_ts/classes/connectionmanager.js +5 -28
  8. package/dist_ts/classes/gitopsapp.d.ts +8 -3
  9. package/dist_ts/classes/gitopsapp.js +29 -18
  10. package/dist_ts/{cache/classes.secrets.scan.service.d.ts → classes/secrets.scan.service.d.ts} +1 -1
  11. package/dist_ts/{cache/classes.secrets.scan.service.js → classes/secrets.scan.service.js} +3 -3
  12. package/dist_ts/{cache → db}/classes.cache.cleaner.d.ts +1 -3
  13. package/dist_ts/db/classes.cache.cleaner.js +59 -0
  14. package/dist_ts/{cache → db}/classes.cached.document.d.ts +1 -1
  15. package/dist_ts/{cache → db}/classes.cached.document.js +2 -2
  16. package/dist_ts/db/classes.gitops-db.d.ts +34 -0
  17. package/dist_ts/db/classes.gitops-db.js +77 -0
  18. package/dist_ts/{cache → db}/documents/classes.cached.project.js +3 -3
  19. package/dist_ts/{cache → db}/documents/classes.cached.secret.js +3 -3
  20. package/dist_ts/db/documents/classes.storage.record.d.ts +14 -0
  21. package/dist_ts/db/documents/classes.storage.record.js +82 -0
  22. package/dist_ts/{cache → db}/documents/index.d.ts +1 -0
  23. package/dist_ts/db/documents/index.js +4 -0
  24. package/dist_ts/db/index.d.ts +5 -0
  25. package/dist_ts/db/index.js +5 -0
  26. package/dist_ts/paths.d.ts +2 -2
  27. package/dist_ts/paths.js +2 -3
  28. package/dist_ts/plugins.d.ts +2 -3
  29. package/dist_ts/plugins.js +6 -4
  30. package/dist_ts/storage/classes.storagemanager.d.ts +7 -16
  31. package/dist_ts/storage/classes.storagemanager.js +54 -94
  32. package/dist_ts/storage/index.d.ts +0 -1
  33. package/dist_ts_migrations/classes.migration-record.d.ts +11 -0
  34. package/dist_ts_migrations/classes.migration-record.js +80 -0
  35. package/dist_ts_migrations/import-filesystem-records.d.ts +9 -0
  36. package/dist_ts_migrations/import-filesystem-records.js +74 -0
  37. package/dist_ts_migrations/index.d.ts +12 -0
  38. package/dist_ts_migrations/index.js +24 -0
  39. package/dist_ts_migrations/interfaces.d.ts +16 -0
  40. package/dist_ts_migrations/interfaces.js +2 -0
  41. package/package.json +6 -4
  42. package/readme.md +21 -14
  43. package/ts/00_commitinfo_data.ts +1 -1
  44. package/ts/classes/actionlog.ts +1 -1
  45. package/ts/classes/connectionmanager.ts +4 -27
  46. package/ts/classes/gitopsapp.ts +36 -17
  47. package/ts/{cache/classes.secrets.scan.service.ts → classes/secrets.scan.service.ts} +3 -3
  48. package/ts/{cache → db}/classes.cache.cleaner.ts +1 -4
  49. package/ts/{cache → db}/classes.cached.document.ts +1 -1
  50. package/ts/db/classes.gitops-db.ts +92 -0
  51. package/ts/{cache → db}/documents/classes.cached.project.ts +2 -2
  52. package/ts/{cache → db}/documents/classes.cached.secret.ts +2 -2
  53. package/ts/db/documents/classes.storage.record.ts +26 -0
  54. package/ts/{cache → db}/documents/index.ts +1 -0
  55. package/ts/db/index.ts +5 -0
  56. package/ts/paths.ts +3 -4
  57. package/ts/plugins.ts +5 -3
  58. package/ts/storage/classes.storagemanager.ts +59 -95
  59. package/ts/storage/index.ts +0 -1
  60. package/ts_migrations/classes.migration-record.ts +23 -0
  61. package/ts_migrations/import-filesystem-records.ts +79 -0
  62. package/ts_migrations/index.ts +31 -0
  63. package/ts_migrations/interfaces.ts +18 -0
  64. package/ts_web/00_commitinfo_data.ts +1 -1
  65. package/dist_ts/cache/classes.cache.cleaner.js +0 -61
  66. package/dist_ts/cache/classes.cachedb.d.ts +0 -22
  67. package/dist_ts/cache/classes.cachedb.js +0 -58
  68. package/dist_ts/cache/documents/index.js +0 -3
  69. package/dist_ts/cache/index.d.ts +0 -7
  70. package/dist_ts/cache/index.js +0 -6
  71. package/ts/cache/classes.cachedb.ts +0 -72
  72. package/ts/cache/index.ts +0 -7
  73. /package/dist_ts/{cache → db}/documents/classes.cached.project.d.ts +0 -0
  74. /package/dist_ts/{cache → db}/documents/classes.cached.secret.d.ts +0 -0
@@ -0,0 +1,92 @@
1
+ import * as plugins from '../plugins.js';
2
+ import { logger } from '../logging.js';
3
+
4
+ export interface IGitopsDbOptions {
5
+ /** Connection URL of the document database gitops persists into. */
6
+ mongoDbUrl: string;
7
+ /** Database name inside that server. */
8
+ mongoDbName: string;
9
+ }
10
+
11
+ const requireEnv = (keyArg: string): string => {
12
+ const value = process.env[keyArg];
13
+ if (!value) {
14
+ throw new Error(`GitOps requires ${keyArg} to be set and non-empty.`);
15
+ }
16
+ if (value.trim() !== value) {
17
+ throw new Error(`GitOps requires ${keyArg} to be canonical without surrounding whitespace.`);
18
+ }
19
+ return value;
20
+ };
21
+
22
+ /**
23
+ * Reads the database binding from the process environment. There is no default and no
24
+ * embedded fallback: gitops keeps every durable record — connections, jobs, sync
25
+ * configuration, managed secrets, the action log and the provider caches — in this one
26
+ * database, so a missing binding has to stop startup rather than silently produce an
27
+ * instance that forgets everything it was told.
28
+ */
29
+ export const readGitopsDbOptionsFromEnvironment = (): IGitopsDbOptions => ({
30
+ mongoDbUrl: requireEnv('GITOPS_MONGODB_URL'),
31
+ // An explicitly set but empty name is a misconfiguration, not a request for the default.
32
+ mongoDbName: process.env.GITOPS_MONGODB_NAME === undefined ? 'gitops' : requireEnv('GITOPS_MONGODB_NAME'),
33
+ });
34
+
35
+ /**
36
+ * The single document database behind gitops.
37
+ *
38
+ * Every model binds its collection through `GitopsDb.getInstance().getDb()`, so the
39
+ * singleton is the only place that knows the connection. Tests construct it with an
40
+ * explicit binding to an isolated server; the application reads the binding from the
41
+ * environment.
42
+ */
43
+ export class GitopsDb {
44
+ private static instance: GitopsDb | null = null;
45
+
46
+ private smartdataDb: plugins.smartdata.SmartdataDb | null = null;
47
+ private readonly options: IGitopsDbOptions;
48
+
49
+ private constructor(options: IGitopsDbOptions) {
50
+ this.options = options;
51
+ }
52
+
53
+ static getInstance(options?: IGitopsDbOptions): GitopsDb {
54
+ if (!GitopsDb.instance) {
55
+ if (!options) {
56
+ throw new Error('GitopsDb has no instance yet — the first call must supply its options.');
57
+ }
58
+ GitopsDb.instance = new GitopsDb(options);
59
+ }
60
+ return GitopsDb.instance;
61
+ }
62
+
63
+ static resetInstance(): void {
64
+ GitopsDb.instance = null;
65
+ }
66
+
67
+ async start(): Promise<void> {
68
+ logger.info('Connecting to the gitops database...');
69
+ const smartdataDb = new plugins.smartdata.SmartdataDb({
70
+ mongoDbUrl: this.options.mongoDbUrl,
71
+ mongoDbName: this.options.mongoDbName,
72
+ });
73
+ await smartdataDb.init();
74
+ this.smartdataDb = smartdataDb;
75
+ logger.success(`Connected to the gitops database (db: ${this.options.mongoDbName})`);
76
+ }
77
+
78
+ async stop(): Promise<void> {
79
+ if (!this.smartdataDb) return;
80
+ const smartdataDb = this.smartdataDb;
81
+ this.smartdataDb = null;
82
+ await smartdataDb.close();
83
+ logger.success('Disconnected from the gitops database');
84
+ }
85
+
86
+ getDb(): plugins.smartdata.SmartdataDb {
87
+ if (!this.smartdataDb) {
88
+ throw new Error('GitopsDb not started. Call start() first.');
89
+ }
90
+ return this.smartdataDb;
91
+ }
92
+ }
@@ -1,11 +1,11 @@
1
1
  import * as plugins from '../../plugins.js';
2
- import { CacheDb } from '../classes.cachedb.js';
2
+ import { GitopsDb } from '../classes.gitops-db.js';
3
3
  import { CachedDocument, TTL } from '../classes.cached.document.js';
4
4
 
5
5
  /**
6
6
  * Cached project data from git providers. TTL: 5 minutes.
7
7
  */
8
- @plugins.smartdata.Collection(() => CacheDb.getInstance().getDb())
8
+ @plugins.smartdata.Collection(() => GitopsDb.getInstance().getDb())
9
9
  export class CachedProject extends CachedDocument<CachedProject> {
10
10
  @plugins.smartdata.unI()
11
11
  public id: string = '';
@@ -1,12 +1,12 @@
1
1
  import * as plugins from '../../plugins.js';
2
- import { CacheDb } from '../classes.cachedb.js';
2
+ import { GitopsDb } from '../classes.gitops-db.js';
3
3
  import { CachedDocument, TTL } from '../classes.cached.document.js';
4
4
  import type { ISecret } from '../../../ts_interfaces/data/secret.js';
5
5
 
6
6
  /**
7
7
  * Cached secret data from git providers. TTL: 24 hours.
8
8
  */
9
- @plugins.smartdata.Collection(() => CacheDb.getInstance().getDb())
9
+ @plugins.smartdata.Collection(() => GitopsDb.getInstance().getDb())
10
10
  export class CachedSecret extends CachedDocument<CachedSecret> {
11
11
  @plugins.smartdata.unI()
12
12
  public id: string = '';
@@ -0,0 +1,26 @@
1
+ import * as plugins from '../../plugins.js';
2
+ import { GitopsDb } from '../classes.gitops-db.js';
3
+
4
+ const getDb = () => GitopsDb.getInstance().getDb();
5
+
6
+ /**
7
+ * One durable key/value record.
8
+ *
9
+ * `key` owns the stored `_id`, so the primary key is the single uniqueness authority: a
10
+ * write is one identity-anchored upsert with no second index to disagree with, and a
11
+ * prefix listing is answered by the `_id` index instead of a collection scan. That is what
12
+ * `identityAsDocumentId: 'key'` declares, and it holds for every record this collection
13
+ * ever received, because the store below has always derived `_id` from the key.
14
+ */
15
+ @plugins.smartdata.Collection(() => getDb(), { identityAsDocumentId: 'key' })
16
+ export class StorageRecord extends plugins.smartdata.SmartDataDbDoc<
17
+ StorageRecord,
18
+ StorageRecord
19
+ > {
20
+ @plugins.smartdata.unI()
21
+ @plugins.smartdata.svDb()
22
+ public key!: string;
23
+
24
+ @plugins.smartdata.svDb()
25
+ public value!: string;
26
+ }
@@ -1,2 +1,3 @@
1
1
  export { CachedProject } from './classes.cached.project.js';
2
2
  export { CachedSecret } from './classes.cached.secret.js';
3
+ export { StorageRecord } from './classes.storage.record.js';
package/ts/db/index.ts ADDED
@@ -0,0 +1,5 @@
1
+ export { GitopsDb, readGitopsDbOptionsFromEnvironment } from './classes.gitops-db.js';
2
+ export type { IGitopsDbOptions } from './classes.gitops-db.js';
3
+ export { CachedDocument, TTL } from './classes.cached.document.js';
4
+ export { CacheCleaner } from './classes.cache.cleaner.js';
5
+ export * from './documents/index.js';
package/ts/paths.ts CHANGED
@@ -2,8 +2,8 @@ import * as plugins from './plugins.js';
2
2
 
3
3
  export interface IGitopsPaths {
4
4
  gitopsHomeDir: string;
5
- defaultStoragePath: string;
6
- defaultTsmDbPath: string;
5
+ /** Directory the releases before the document database wrote key/value records into. */
6
+ legacyStoragePath: string;
7
7
  }
8
8
 
9
9
  /**
@@ -13,7 +13,6 @@ export function resolvePaths(baseDir?: string): IGitopsPaths {
13
13
  const home = baseDir ?? plugins.path.join(process.env.HOME ?? '/tmp', '.serve.zone', 'gitops');
14
14
  return {
15
15
  gitopsHomeDir: home,
16
- defaultStoragePath: plugins.path.join(home, 'storage'),
17
- defaultTsmDbPath: plugins.path.join(home, 'tsmdb'),
16
+ legacyStoragePath: plugins.path.join(home, 'storage'),
18
17
  };
19
18
  }
package/ts/plugins.ts CHANGED
@@ -30,9 +30,11 @@ import * as bookstackClient from '@apiclient.xyz/bookstack';
30
30
  export { giteaClient, gitlabClient, bookstackClient };
31
31
 
32
32
  // Database
33
- import * as smartmongo from '@push.rocks/smartmongo';
34
- import * as smartdata from '@push.rocks/smartdata';
35
- export { smartmongo, smartdata };
33
+ // The document ORM. Exported as `smartdata` because that is the name every call site uses and
34
+ // because `@lossless.org/client/nosqldb` supersedes `@push.rocks/smartdata` with an identical
35
+ // API surface; the persisted shape is unchanged.
36
+ import * as smartdata from '@lossless.org/client/nosqldb';
37
+ export { smartdata };
36
38
 
37
39
  // Secrets
38
40
  import * as smartsecret from '@push.rocks/smartsecret';
@@ -1,31 +1,27 @@
1
- import * as plugins from '../plugins.js';
1
+ import { StorageRecord } from '../db/documents/classes.storage.record.js';
2
2
 
3
- export type TStorageBackend = 'filesystem' | 'memory';
4
-
5
- export interface IStorageConfig {
6
- backend?: TStorageBackend;
7
- fsPath?: string;
8
- }
3
+ /**
4
+ * The upper bound of the key range scanned for a prefix listing. `￿` is above every
5
+ * character a normalized key may contain, so `[prefix/, prefix/￿)` covers the whole
6
+ * subtree of a prefix and nothing beyond it.
7
+ */
8
+ const KEY_RANGE_END = '￿';
9
9
 
10
- const isNotFoundError = (error: unknown): boolean => {
11
- return error instanceof Error && (error as NodeJS.ErrnoException).code === 'ENOENT';
12
- };
10
+ /**
11
+ * The key is the record's primary key, and MongoDB indexes a primary key of at most
12
+ * 1024 bytes; the bound is stated here so an oversized key is refused by name instead of
13
+ * surfacing as a driver index error on write.
14
+ */
15
+ const MAX_KEY_BYTES = 1024;
13
16
 
14
17
  /**
15
- * Key-value storage abstraction with filesystem and memory backends.
16
- * Keys must start with '/' and are normalized (no '..', no double slashes).
18
+ * Key-value storage for gitops' durable records, kept in the `StorageRecord` collection.
19
+ *
20
+ * Keys must start with '/' and are normalized (no '..', no double slashes). Listing is
21
+ * non-recursive: it returns the direct children of a prefix, which is what every caller
22
+ * relies on to keep neighbouring key spaces apart.
17
23
  */
18
24
  export class StorageManager {
19
- private backend: TStorageBackend;
20
- private fsPath: string;
21
- private memoryStore: Map<string, string>;
22
-
23
- constructor(config: IStorageConfig = {}) {
24
- this.backend = config.backend ?? 'filesystem';
25
- this.fsPath = config.fsPath ?? './storage';
26
- this.memoryStore = new Map();
27
- }
28
-
29
25
  /**
30
26
  * Normalize and validate a storage key.
31
27
  */
@@ -35,101 +31,69 @@ export class StorageManager {
35
31
  }
36
32
  // Strip '..' segments and normalize double slashes
37
33
  const segments = key.split('/').filter((s) => s !== '' && s !== '..');
38
- return '/' + segments.join('/');
39
- }
40
-
41
- /**
42
- * Resolve a key to a filesystem path.
43
- */
44
- private keyToPath(key: string): string {
45
- const normalized = this.normalizeKey(key);
46
- return plugins.path.join(this.fsPath, ...normalized.split('/').filter(Boolean));
34
+ const normalized = '/' + segments.join('/');
35
+ if (Buffer.byteLength(normalized, 'utf8') > MAX_KEY_BYTES) {
36
+ throw new Error(`Storage key exceeds ${MAX_KEY_BYTES} bytes: ${normalized.slice(0, 64)}…`);
37
+ }
38
+ return normalized;
47
39
  }
48
40
 
49
41
  async get(key: string): Promise<string | null> {
50
- const normalized = this.normalizeKey(key);
51
- if (this.backend === 'memory') {
52
- return this.memoryStore.get(normalized) ?? null;
53
- }
54
- try {
55
- return await plugins.fs.readFile(this.keyToPath(normalized), 'utf8');
56
- } catch (err) {
57
- if (isNotFoundError(err)) return null;
58
- throw err;
59
- }
42
+ const record = await StorageRecord.getInstance<StorageRecord>({
43
+ key: this.normalizeKey(key),
44
+ });
45
+ return record ? record.value : null;
60
46
  }
61
47
 
62
48
  async set(key: string, value: string): Promise<void> {
63
- const normalized = this.normalizeKey(key);
64
- if (this.backend === 'memory') {
65
- this.memoryStore.set(normalized, value);
66
- return;
67
- }
68
- const filePath = this.keyToPath(normalized);
69
- const dir = plugins.path.dirname(filePath);
70
- await plugins.fs.mkdir(dir, { recursive: true });
71
- // Atomic write: write to temp then rename
72
- const tmpPath = filePath + '.tmp';
73
- await plugins.fs.writeFile(tmpPath, value, 'utf8');
74
- await plugins.fs.rename(tmpPath, filePath);
49
+ // The equality anchor on `key` is what makes this a single race-free upsert: the
50
+ // primary key is derived from the filter, and the client seeds the immutable identity
51
+ // from that same anchor, so the key is never restated in the update document.
52
+ await StorageRecord.atomicUpdate<StorageRecord>(
53
+ { key: this.normalizeKey(key) },
54
+ { $set: { value } },
55
+ { upsert: true },
56
+ );
75
57
  }
76
58
 
77
59
  async delete(key: string): Promise<boolean> {
78
60
  const normalized = this.normalizeKey(key);
79
- if (this.backend === 'memory') {
80
- return this.memoryStore.delete(normalized);
81
- }
82
- try {
83
- await plugins.fs.rm(this.keyToPath(normalized));
84
- return true;
85
- } catch (err) {
86
- if (isNotFoundError(err)) return false;
87
- throw err;
61
+ if (!(await StorageRecord.exists<StorageRecord>({ key: normalized }))) {
62
+ return false;
88
63
  }
64
+ // The identity pins a single document, which is what a singular delete requires.
65
+ await StorageRecord.atomicDelete<StorageRecord>({ key: normalized });
66
+ return true;
89
67
  }
90
68
 
91
69
  async exists(key: string): Promise<boolean> {
92
- const normalized = this.normalizeKey(key);
93
- if (this.backend === 'memory') {
94
- return this.memoryStore.has(normalized);
95
- }
96
- try {
97
- await plugins.fs.stat(this.keyToPath(normalized));
98
- return true;
99
- } catch (err) {
100
- if (isNotFoundError(err)) return false;
101
- throw err;
102
- }
70
+ return StorageRecord.exists<StorageRecord>({ key: this.normalizeKey(key) });
103
71
  }
104
72
 
105
73
  /**
106
- * List keys under a given prefix.
74
+ * List the keys directly below a given prefix. Records deeper in the subtree are read by
75
+ * the range scan and then dropped, mirroring the directory listing this replaced.
107
76
  */
108
77
  async list(prefix: string): Promise<string[]> {
109
78
  const normalized = this.normalizeKey(prefix);
110
- if (this.backend === 'memory') {
111
- const keys: string[] = [];
112
- for (const key of this.memoryStore.keys()) {
113
- if (key.startsWith(normalized)) {
114
- keys.push(key);
115
- }
116
- }
117
- return keys.sort();
118
- }
119
- const dirPath = this.keyToPath(normalized);
120
- const keys: string[] = [];
79
+ // Normalization leaves the root as '/', which is already its own child prefix.
80
+ const childPrefix = normalized.endsWith('/') ? normalized : `${normalized}/`;
81
+ const cursor = await StorageRecord.getCursor<StorageRecord>(
82
+ { key: { $gte: childPrefix, $lt: `${childPrefix}${KEY_RANGE_END}` } },
83
+ // Only the keys are wanted here; leaving the values in the database keeps a listing
84
+ // independent of how much each record holds.
85
+ { projection: { key: 1 } },
86
+ );
87
+ let records: StorageRecord[];
121
88
  try {
122
- const entries = await plugins.fs.readdir(dirPath, { withFileTypes: true });
123
- for (const entry of entries) {
124
- if (entry.isFile()) {
125
- keys.push(normalized.replace(/\/$/, '') + '/' + entry.name);
126
- }
127
- }
128
- } catch (err) {
129
- if (isNotFoundError(err)) return [];
130
- throw err;
89
+ records = await cursor.toArray();
90
+ } finally {
91
+ await cursor.close();
131
92
  }
132
- return keys.sort();
93
+ return records
94
+ .map((recordArg) => recordArg.key)
95
+ .filter((keyArg) => !keyArg.slice(childPrefix.length).includes('/'))
96
+ .sort();
133
97
  }
134
98
 
135
99
  async getJSON<T>(key: string): Promise<T | null> {
@@ -1,2 +1 @@
1
1
  export { StorageManager } from './classes.storagemanager.js';
2
- export type { IStorageConfig, TStorageBackend } from './classes.storagemanager.js';
@@ -0,0 +1,23 @@
1
+ import * as plugins from '../ts/plugins.js';
2
+ import { GitopsDb } from '../ts/db/index.js';
3
+
4
+ /**
5
+ * The ledger of migrations that have run to completion.
6
+ *
7
+ * `migration` owns the stored `_id`, so a migration is looked up by primary key and can
8
+ * only ever be recorded once.
9
+ */
10
+ @plugins.smartdata.Collection(() => GitopsDb.getInstance().getDb(), {
11
+ identityAsDocumentId: 'migration',
12
+ })
13
+ export class MigrationRecord extends plugins.smartdata.SmartDataDbDoc<
14
+ MigrationRecord,
15
+ MigrationRecord
16
+ > {
17
+ @plugins.smartdata.unI()
18
+ @plugins.smartdata.svDb()
19
+ public migration!: string;
20
+
21
+ @plugins.smartdata.svDb()
22
+ public completedAt!: number;
23
+ }
@@ -0,0 +1,79 @@
1
+ import * as plugins from '../ts/plugins.js';
2
+ import type { IMigration, IMigrationContext } from './interfaces.js';
3
+
4
+ interface ILegacyConnection {
5
+ id: string;
6
+ }
7
+
8
+ /**
9
+ * Reads every regular file below `directoryArg` and yields it as a storage key relative to
10
+ * the tree root, so `<root>/connections/a.json` becomes `/connections/a.json`.
11
+ */
12
+ const readDirectoryEntries = async (directoryArg: string) => {
13
+ try {
14
+ return await plugins.fs.readdir(directoryArg, { withFileTypes: true });
15
+ } catch (error) {
16
+ if ((error as NodeJS.ErrnoException).code === 'ENOENT') return [];
17
+ throw error;
18
+ }
19
+ };
20
+
21
+ const readLegacyTree = async (
22
+ directoryArg: string,
23
+ prefixArg: string,
24
+ ): Promise<{ key: string; value: string }[]> => {
25
+ const entries = await readDirectoryEntries(directoryArg);
26
+ const records: { key: string; value: string }[] = [];
27
+ for (const entry of entries) {
28
+ const entryPath = plugins.path.join(directoryArg, entry.name);
29
+ if (entry.isDirectory()) {
30
+ records.push(...(await readLegacyTree(entryPath, `${prefixArg}/${entry.name}`)));
31
+ } else if (entry.isFile()) {
32
+ records.push({
33
+ key: `${prefixArg}/${entry.name}`,
34
+ value: await plugins.fs.readFile(entryPath, 'utf8'),
35
+ });
36
+ }
37
+ }
38
+ return records;
39
+ };
40
+
41
+ /**
42
+ * Imports the key/value records that earlier releases kept as files.
43
+ *
44
+ * Both sources are read once and never written back: the ledger, not a renamed file, is
45
+ * what records that this ran. The legacy tree is left on disk untouched so an operator can
46
+ * archive it after confirming the import.
47
+ */
48
+ export const importFilesystemRecords: IMigration = {
49
+ name: 'import-filesystem-records',
50
+ run: async (contextArg: IMigrationContext): Promise<void> => {
51
+ // The development-time connections file was the authority over the tree before, and
52
+ // stays it here: it is written first and the tree import below skips existing keys.
53
+ try {
54
+ const legacyText = await plugins.fs.readFile(contextArg.legacyConnectionsFile, 'utf8');
55
+ const legacyConnections = JSON.parse(legacyText) as ILegacyConnection[];
56
+ for (const connection of legacyConnections) {
57
+ await contextArg.storage.setJSON(`/connections/${connection.id}.json`, connection);
58
+ }
59
+ if (legacyConnections.length > 0) {
60
+ contextArg.log(`imported ${legacyConnections.length} connection(s) from the legacy file`);
61
+ }
62
+ } catch (error) {
63
+ if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
64
+ }
65
+
66
+ const records = await readLegacyTree(contextArg.legacyStoragePath, '');
67
+ let imported = 0;
68
+ for (const record of records) {
69
+ // A key that already exists was written by a newer authority — the legacy file above
70
+ // or a release that already ran against the database — so it is never overwritten.
71
+ if (await contextArg.storage.exists(record.key)) continue;
72
+ await contextArg.storage.set(record.key, record.value);
73
+ imported++;
74
+ }
75
+ if (imported > 0) {
76
+ contextArg.log(`imported ${imported} record(s) from ${contextArg.legacyStoragePath}`);
77
+ }
78
+ },
79
+ };
@@ -0,0 +1,31 @@
1
+ import { MigrationRecord } from './classes.migration-record.js';
2
+ import { importFilesystemRecords } from './import-filesystem-records.js';
3
+ import type { IMigration, IMigrationContext } from './interfaces.js';
4
+
5
+ export type { IMigration, IMigrationContext } from './interfaces.js';
6
+ export { MigrationRecord } from './classes.migration-record.js';
7
+
8
+ /** Applied in order; a name that is already in the ledger is skipped. */
9
+ const migrations: readonly IMigration[] = [importFilesystemRecords];
10
+
11
+ /**
12
+ * Runs the pending migrations against an already connected database.
13
+ *
14
+ * Called from startup only, after the database is up and before any manager reads a
15
+ * record. No lease is taken: every migration is idempotent, so two instances starting at
16
+ * the same time converge on the same result, while a lease would let one crashed claimant
17
+ * block every later start.
18
+ */
19
+ export const runStartupMigrations = async (contextArg: IMigrationContext): Promise<void> => {
20
+ for (const migration of migrations) {
21
+ if (await MigrationRecord.exists<MigrationRecord>({ migration: migration.name })) continue;
22
+ contextArg.log(`running migration ${migration.name}`);
23
+ await migration.run(contextArg);
24
+ await MigrationRecord.atomicUpdate<MigrationRecord>(
25
+ { migration: migration.name },
26
+ { $set: { completedAt: Date.now() } },
27
+ { upsert: true },
28
+ );
29
+ contextArg.log(`migration ${migration.name} completed`);
30
+ }
31
+ };
@@ -0,0 +1,18 @@
1
+ import type { StorageManager } from '../ts/storage/index.js';
2
+
3
+ export interface IMigrationContext {
4
+ /** The document-backed key/value store every migration writes through. */
5
+ storage: StorageManager;
6
+ /** Root of the key/value tree the releases before the document database wrote. */
7
+ legacyStoragePath: string;
8
+ /** The development-time connection list earlier releases imported on every start. */
9
+ legacyConnectionsFile: string;
10
+ /** Reports progress to the application log. */
11
+ log: (messageArg: string) => void;
12
+ }
13
+
14
+ export interface IMigration {
15
+ /** Stable identity recorded in the ledger once the migration completed. */
16
+ name: string;
17
+ run: (contextArg: IMigrationContext) => Promise<void>;
18
+ }
@@ -3,6 +3,6 @@
3
3
  */
4
4
  export const commitinfo = {
5
5
  name: '@serve.zone/gitops',
6
- version: '32.1.0',
6
+ version: '32.2.0',
7
7
  description: 'GitOps management app for Gitea and GitLab - manage secrets, browse projects, view CI pipelines, and stream build logs'
8
8
  }
@@ -1,61 +0,0 @@
1
- import { logger } from '../logging.js';
2
- import { unrefTimer } from '../timers.js';
3
- const DEFAULT_INTERVAL_MS = 60 * 60 * 1000; // 1 hour
4
- /**
5
- * Periodically cleans up expired cached documents.
6
- */
7
- export class CacheCleaner {
8
- intervalId = null;
9
- intervalMs;
10
- documentClasses = [];
11
- cacheDb;
12
- constructor(cacheDb, intervalMs = DEFAULT_INTERVAL_MS) {
13
- this.cacheDb = cacheDb;
14
- this.intervalMs = intervalMs;
15
- }
16
- /** Register a document class for cleanup */
17
- registerClass(cls) {
18
- this.documentClasses.push(cls);
19
- }
20
- start() {
21
- if (this.intervalId !== null)
22
- return;
23
- this.intervalId = setInterval(() => {
24
- this.clean().catch((err) => {
25
- logger.error(`CacheCleaner error: ${err}`);
26
- });
27
- }, this.intervalMs);
28
- // Unref so the interval doesn't prevent process exit
29
- unrefTimer(this.intervalId);
30
- logger.debug(`CacheCleaner started (interval: ${this.intervalMs}ms)`);
31
- }
32
- stop() {
33
- if (this.intervalId !== null) {
34
- clearInterval(this.intervalId);
35
- this.intervalId = null;
36
- logger.debug('CacheCleaner stopped');
37
- }
38
- }
39
- /** Run a single cleanup pass */
40
- async clean() {
41
- const now = Date.now();
42
- let totalDeleted = 0;
43
- for (const cls of this.documentClasses) {
44
- try {
45
- const expired = await cls.getInstances({ expiresAt: { $lt: now } });
46
- for (const doc of expired) {
47
- await doc.delete();
48
- totalDeleted++;
49
- }
50
- }
51
- catch (err) {
52
- logger.error(`CacheCleaner: failed to clean class: ${err}`);
53
- }
54
- }
55
- if (totalDeleted > 0) {
56
- logger.debug(`CacheCleaner: deleted ${totalDeleted} expired document(s)`);
57
- }
58
- return totalDeleted;
59
- }
60
- }
61
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY2xhc3Nlcy5jYWNoZS5jbGVhbmVyLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vdHMvY2FjaGUvY2xhc3Nlcy5jYWNoZS5jbGVhbmVyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxNQUFNLEVBQUUsTUFBTSxlQUFlLENBQUM7QUFDdkMsT0FBTyxFQUFFLFVBQVUsRUFBRSxNQUFNLGNBQWMsQ0FBQztBQU0xQyxNQUFNLG1CQUFtQixHQUFHLEVBQUUsR0FBRyxFQUFFLEdBQUcsSUFBSSxDQUFDLENBQUMsU0FBUztBQUVyRDs7R0FFRztBQUNILE1BQU0sT0FBTyxZQUFZO0lBQ2YsVUFBVSxHQUEwQyxJQUFJLENBQUM7SUFDekQsVUFBVSxDQUFTO0lBQ25CLGVBQWUsR0FBb0IsRUFBRSxDQUFDO0lBQ3RDLE9BQU8sQ0FBVTtJQUV6QixZQUFZLE9BQWdCLEVBQUUsVUFBVSxHQUFHLG1CQUFtQjtRQUM1RCxJQUFJLENBQUMsT0FBTyxHQUFHLE9BQU8sQ0FBQztRQUN2QixJQUFJLENBQUMsVUFBVSxHQUFHLFVBQVUsQ0FBQztJQUMvQixDQUFDO0lBRUQsNENBQTRDO0lBQzVDLGFBQWEsQ0FBQyxHQUFrQjtRQUM5QixJQUFJLENBQUMsZUFBZSxDQUFDLElBQUksQ0FBQyxHQUFHLENBQUMsQ0FBQztJQUNqQyxDQUFDO0lBRUQsS0FBSztRQUNILElBQUksSUFBSSxDQUFDLFVBQVUsS0FBSyxJQUFJO1lBQUUsT0FBTztRQUNyQyxJQUFJLENBQUMsVUFBVSxHQUFHLFdBQVcsQ0FBQyxHQUFHLEVBQUU7WUFDakMsSUFBSSxDQUFDLEtBQUssRUFBRSxDQUFDLEtBQUssQ0FBQyxDQUFDLEdBQUcsRUFBRSxFQUFFO2dCQUN6QixNQUFNLENBQUMsS0FBSyxDQUFDLHVCQUF1QixHQUFHLEVBQUUsQ0FBQyxDQUFDO1lBQzdDLENBQUMsQ0FBQyxDQUFDO1FBQ0wsQ0FBQyxFQUFFLElBQUksQ0FBQyxVQUFVLENBQUMsQ0FBQztRQUNwQixxREFBcUQ7UUFDckQsVUFBVSxDQUFDLElBQUksQ0FBQyxVQUFVLENBQUMsQ0FBQztRQUM1QixNQUFNLENBQUMsS0FBSyxDQUFDLG1DQUFtQyxJQUFJLENBQUMsVUFBVSxLQUFLLENBQUMsQ0FBQztJQUN4RSxDQUFDO0lBRUQsSUFBSTtRQUNGLElBQUksSUFBSSxDQUFDLFVBQVUsS0FBSyxJQUFJLEVBQUUsQ0FBQztZQUM3QixhQUFhLENBQUMsSUFBSSxDQUFDLFVBQVUsQ0FBQyxDQUFDO1lBQy9CLElBQUksQ0FBQyxVQUFVLEdBQUcsSUFBSSxDQUFDO1lBQ3ZCLE1BQU0sQ0FBQyxLQUFLLENBQUMsc0JBQXNCLENBQUMsQ0FBQztRQUN2QyxDQUFDO0lBQ0gsQ0FBQztJQUVELGdDQUFnQztJQUNoQyxLQUFLLENBQUMsS0FBSztRQUNULE1BQU0sR0FBRyxHQUFHLElBQUksQ0FBQyxHQUFHLEVBQUUsQ0FBQztRQUN2QixJQUFJLFlBQVksR0FBRyxDQUFDLENBQUM7UUFDckIsS0FBSyxNQUFNLEdBQUcsSUFBSSxJQUFJLENBQUMsZUFBZSxFQUFFLENBQUM7WUFDdkMsSUFBSSxDQUFDO2dCQUNILE1BQU0sT0FBTyxHQUFHLE1BQU0sR0FBRyxDQUFDLFlBQVksQ0FBQyxFQUFFLFNBQVMsRUFBRSxFQUFFLEdBQUcsRUFBRSxHQUFHLEVBQUUsRUFBRSxDQUFDLENBQUM7Z0JBQ3BFLEtBQUssTUFBTSxHQUFHLElBQUksT0FBTyxFQUFFLENBQUM7b0JBQzFCLE1BQU0sR0FBRyxDQUFDLE1BQU0sRUFBRSxDQUFDO29CQUNuQixZQUFZLEVBQUUsQ0FBQztnQkFDakIsQ0FBQztZQUNILENBQUM7WUFBQyxPQUFPLEdBQUcsRUFBRSxDQUFDO2dCQUNiLE1BQU0sQ0FBQyxLQUFLLENBQUMsd0NBQXdDLEdBQUcsRUFBRSxDQUFDLENBQUM7WUFDOUQsQ0FBQztRQUNILENBQUM7UUFDRCxJQUFJLFlBQVksR0FBRyxDQUFDLEVBQUUsQ0FBQztZQUNyQixNQUFNLENBQUMsS0FBSyxDQUFDLHlCQUF5QixZQUFZLHNCQUFzQixDQUFDLENBQUM7UUFDNUUsQ0FBQztRQUNELE9BQU8sWUFBWSxDQUFDO0lBQ3RCLENBQUM7Q0FDRiJ9
@@ -1,22 +0,0 @@
1
- import * as plugins from '../plugins.js';
2
- export interface ICacheDbOptions {
3
- storagePath?: string;
4
- dbName?: string;
5
- debug?: boolean;
6
- }
7
- /**
8
- * Singleton wrapper around SmartMongo's NoSQLDB memory server and SmartData.
9
- * Provides a managed MongoDB-compatible cache database.
10
- */
11
- export declare class CacheDb {
12
- private static instance;
13
- private smartMongo;
14
- private smartdataDb;
15
- private options;
16
- private constructor();
17
- static getInstance(options?: ICacheDbOptions): CacheDb;
18
- static resetInstance(): void;
19
- start(): Promise<void>;
20
- stop(): Promise<void>;
21
- getDb(): InstanceType<typeof plugins.smartdata.SmartdataDb>;
22
- }