@checkstack/gitops-backend 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -0
- package/drizzle/0000_tense_stryfe.sql +46 -0
- package/drizzle/meta/0000_snapshot.json +310 -0
- package/drizzle/meta/_journal.json +13 -0
- package/drizzle.config.ts +7 -0
- package/package.json +37 -0
- package/src/index.ts +136 -0
- package/src/kind-registry.test.ts +262 -0
- package/src/kind-registry.ts +191 -0
- package/src/router.ts +355 -0
- package/src/schema.ts +77 -0
- package/src/scrapers/github-scraper.test.ts +355 -0
- package/src/scrapers/github-scraper.ts +263 -0
- package/src/scrapers/gitlab-scraper.test.ts +296 -0
- package/src/scrapers/gitlab-scraper.ts +242 -0
- package/src/scrapers/types.ts +52 -0
- package/src/secret-resolver.test.ts +86 -0
- package/src/secret-resolver.ts +54 -0
- package/src/sync/document-parser.test.ts +116 -0
- package/src/sync/document-parser.ts +124 -0
- package/src/sync/reconciler-delete.test.ts +123 -0
- package/src/sync/reconciler.ts +476 -0
- package/src/sync/sort-entities.test.ts +481 -0
- package/src/sync/sort-entities.ts +100 -0
- package/src/sync/sync-worker.ts +158 -0
- package/tsconfig.json +4 -0
package/src/router.ts
ADDED
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
import { implement, ORPCError } from "@orpc/server";
|
|
2
|
+
import { autoAuthMiddleware, type RpcContext } from "@checkstack/backend-api";
|
|
3
|
+
import { encrypt, decrypt } from "@checkstack/backend-api";
|
|
4
|
+
import { gitopsContract } from "@checkstack/gitops-common";
|
|
5
|
+
import type { SafeDatabase } from "@checkstack/backend-api";
|
|
6
|
+
import type { QueueManager } from "@checkstack/queue-api";
|
|
7
|
+
import type { InternalEntityKindRegistry } from "./kind-registry";
|
|
8
|
+
import { triggerSyncForProvider } from "./sync/sync-worker";
|
|
9
|
+
import * as schema from "./schema";
|
|
10
|
+
import { eq, and } from "drizzle-orm";
|
|
11
|
+
import { v4 as uuidv4 } from "uuid";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Creates the GitOps router using contract-based implementation.
|
|
15
|
+
*
|
|
16
|
+
* Auth and access rules are automatically enforced via autoAuthMiddleware
|
|
17
|
+
* based on the contract's meta.userType and meta.access.
|
|
18
|
+
*/
|
|
19
|
+
const os = implement(gitopsContract)
|
|
20
|
+
.$context<RpcContext>()
|
|
21
|
+
.use(autoAuthMiddleware);
|
|
22
|
+
|
|
23
|
+
export interface GitOpsRouterDeps {
|
|
24
|
+
database: SafeDatabase<typeof schema>;
|
|
25
|
+
queueManager: QueueManager;
|
|
26
|
+
kindRegistry: InternalEntityKindRegistry;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export const createGitOpsRouter = ({
|
|
30
|
+
database: db,
|
|
31
|
+
queueManager,
|
|
32
|
+
kindRegistry,
|
|
33
|
+
}: GitOpsRouterDeps) => {
|
|
34
|
+
// ─── Provenance ──────────────────────────────────────────────────────
|
|
35
|
+
|
|
36
|
+
const getProvenance = os.getProvenance.handler(async ({ input }) => {
|
|
37
|
+
const conditions = [eq(schema.provenance.kind, input.kind)];
|
|
38
|
+
|
|
39
|
+
if (input.entityName) {
|
|
40
|
+
conditions.push(eq(schema.provenance.entityName, input.entityName));
|
|
41
|
+
}
|
|
42
|
+
if (input.entityId) {
|
|
43
|
+
conditions.push(eq(schema.provenance.entityId, input.entityId));
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const result = await db
|
|
47
|
+
.select()
|
|
48
|
+
.from(schema.provenance)
|
|
49
|
+
.where(and(...conditions));
|
|
50
|
+
// eslint-disable-next-line unicorn/no-null
|
|
51
|
+
return result[0] ?? null;
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
const listProvenance = os.listProvenance.handler(async ({ input }) => {
|
|
55
|
+
const rows = await db.select().from(schema.provenance);
|
|
56
|
+
if (!input) return rows;
|
|
57
|
+
|
|
58
|
+
return rows.filter((row) => {
|
|
59
|
+
if (input.status && row.status !== input.status) return false;
|
|
60
|
+
if (input.providerId && row.providerId !== input.providerId) return false;
|
|
61
|
+
return true;
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
// ─── Provider Management ─────────────────────────────────────────────
|
|
66
|
+
|
|
67
|
+
const listProviders = os.listProviders.handler(async () => {
|
|
68
|
+
const rows = await db.select().from(schema.providers);
|
|
69
|
+
return rows.map((r) => ({
|
|
70
|
+
id: r.id,
|
|
71
|
+
type: r.type,
|
|
72
|
+
target: r.target,
|
|
73
|
+
pathPattern: r.pathPattern,
|
|
74
|
+
baseUrl: r.baseUrl,
|
|
75
|
+
syncInterval: r.syncInterval,
|
|
76
|
+
deletionPolicy: r.deletionPolicy,
|
|
77
|
+
lastSyncAt: r.lastSyncAt,
|
|
78
|
+
lastSyncError: r.lastSyncError,
|
|
79
|
+
createdAt: r.createdAt,
|
|
80
|
+
}));
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
const createProvider = os.createProvider.handler(async ({ input }) => {
|
|
84
|
+
const id = uuidv4();
|
|
85
|
+
await db.insert(schema.providers).values({
|
|
86
|
+
id,
|
|
87
|
+
type: input.type,
|
|
88
|
+
target: input.target,
|
|
89
|
+
pathPattern: input.pathPattern,
|
|
90
|
+
baseUrl: input.baseUrl ?? null, // eslint-disable-line unicorn/no-null
|
|
91
|
+
authToken: input.authToken ? encrypt(input.authToken) : null, // eslint-disable-line unicorn/no-null
|
|
92
|
+
syncInterval: input.syncInterval ?? 300,
|
|
93
|
+
deletionPolicy: input.deletionPolicy ?? "orphan",
|
|
94
|
+
});
|
|
95
|
+
return { id };
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
const updateProvider = os.updateProvider.handler(async ({ input }) => {
|
|
99
|
+
const existing = await db
|
|
100
|
+
.select()
|
|
101
|
+
.from(schema.providers)
|
|
102
|
+
.where(eq(schema.providers.id, input.id));
|
|
103
|
+
|
|
104
|
+
if (!existing[0]) {
|
|
105
|
+
throw new ORPCError("NOT_FOUND", {
|
|
106
|
+
message: `Provider not found: ${input.id}`,
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const updates: Record<string, unknown> = { updatedAt: new Date() };
|
|
111
|
+
if (input.data.target !== undefined) updates.target = input.data.target;
|
|
112
|
+
if (input.data.pathPattern !== undefined)
|
|
113
|
+
updates.pathPattern = input.data.pathPattern;
|
|
114
|
+
if (input.data.baseUrl !== undefined) updates.baseUrl = input.data.baseUrl;
|
|
115
|
+
if (input.data.authToken !== undefined)
|
|
116
|
+
updates.authToken = encrypt(input.data.authToken);
|
|
117
|
+
if (input.data.syncInterval !== undefined)
|
|
118
|
+
updates.syncInterval = input.data.syncInterval;
|
|
119
|
+
if (input.data.deletionPolicy !== undefined)
|
|
120
|
+
updates.deletionPolicy = input.data.deletionPolicy;
|
|
121
|
+
|
|
122
|
+
await db
|
|
123
|
+
.update(schema.providers)
|
|
124
|
+
.set(updates)
|
|
125
|
+
.where(eq(schema.providers.id, input.id));
|
|
126
|
+
|
|
127
|
+
return { success: true };
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
const deleteProvider = os.deleteProvider.handler(async ({ input }) => {
|
|
131
|
+
const existing = await db
|
|
132
|
+
.select()
|
|
133
|
+
.from(schema.providers)
|
|
134
|
+
.where(eq(schema.providers.id, input.id));
|
|
135
|
+
|
|
136
|
+
if (!existing[0]) {
|
|
137
|
+
throw new ORPCError("NOT_FOUND", {
|
|
138
|
+
message: `Provider not found: ${input.id}`,
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// Provenance entries are cascade-deleted via FK constraint
|
|
143
|
+
await db.delete(schema.providers).where(eq(schema.providers.id, input.id));
|
|
144
|
+
|
|
145
|
+
return { success: true };
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
const triggerSync = os.triggerSync.handler(async ({ input }) => {
|
|
149
|
+
// Verify provider exists
|
|
150
|
+
const provider = await db
|
|
151
|
+
.select()
|
|
152
|
+
.from(schema.providers)
|
|
153
|
+
.where(eq(schema.providers.id, input.providerId));
|
|
154
|
+
|
|
155
|
+
if (!provider[0]) {
|
|
156
|
+
throw new ORPCError("NOT_FOUND", {
|
|
157
|
+
message: `Provider not found: ${input.providerId}`,
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// Dispatch one-off sync job via the queue
|
|
162
|
+
await triggerSyncForProvider({
|
|
163
|
+
queueManager,
|
|
164
|
+
providerId: input.providerId,
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
return { success: true };
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
const confirmOrphanDeletion = os.confirmOrphanDeletion.handler(
|
|
171
|
+
async ({ input }) => {
|
|
172
|
+
const rows = await db
|
|
173
|
+
.select()
|
|
174
|
+
.from(schema.provenance)
|
|
175
|
+
.where(eq(schema.provenance.id, input.provenanceId));
|
|
176
|
+
|
|
177
|
+
const prov = rows[0];
|
|
178
|
+
if (!prov) {
|
|
179
|
+
throw new ORPCError("NOT_FOUND", {
|
|
180
|
+
message: `Provenance entry not found: ${input.provenanceId}`,
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
if (prov.status !== "orphaned") {
|
|
185
|
+
throw new ORPCError("BAD_REQUEST", {
|
|
186
|
+
message: "Only orphaned entities can be confirmed for deletion",
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
// Call the kind's delete reconciler before removing provenance
|
|
191
|
+
const kindDef = kindRegistry.getKind({
|
|
192
|
+
apiVersion: prov.apiVersion,
|
|
193
|
+
kind: prov.kind,
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
if (kindDef?.delete) {
|
|
197
|
+
try {
|
|
198
|
+
await kindDef.delete({
|
|
199
|
+
entityName: prov.entityName,
|
|
200
|
+
entityId: prov.entityId,
|
|
201
|
+
context: {
|
|
202
|
+
logger: {
|
|
203
|
+
debug: () => {},
|
|
204
|
+
info: () => {},
|
|
205
|
+
warn: () => {},
|
|
206
|
+
error: () => {},
|
|
207
|
+
},
|
|
208
|
+
resolveEntityRef: async () => {
|
|
209
|
+
// eslint-disable-next-line unicorn/no-useless-undefined
|
|
210
|
+
return undefined;
|
|
211
|
+
},
|
|
212
|
+
},
|
|
213
|
+
});
|
|
214
|
+
} catch (deleteError) {
|
|
215
|
+
throw new ORPCError("INTERNAL_SERVER_ERROR", {
|
|
216
|
+
message: `Delete reconciler failed: ${deleteError}`,
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
await db
|
|
222
|
+
.delete(schema.provenance)
|
|
223
|
+
.where(eq(schema.provenance.id, input.provenanceId));
|
|
224
|
+
|
|
225
|
+
return { success: true };
|
|
226
|
+
},
|
|
227
|
+
);
|
|
228
|
+
|
|
229
|
+
const dismissOrphan = os.dismissOrphan.handler(async ({ input }) => {
|
|
230
|
+
const rows = await db
|
|
231
|
+
.select()
|
|
232
|
+
.from(schema.provenance)
|
|
233
|
+
.where(eq(schema.provenance.id, input.provenanceId));
|
|
234
|
+
|
|
235
|
+
if (!rows[0]) {
|
|
236
|
+
throw new ORPCError("NOT_FOUND", {
|
|
237
|
+
message: `Provenance entry not found: ${input.provenanceId}`,
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
await db
|
|
242
|
+
.delete(schema.provenance)
|
|
243
|
+
.where(eq(schema.provenance.id, input.provenanceId));
|
|
244
|
+
|
|
245
|
+
return { success: true };
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
// ─── Secret Management ───────────────────────────────────────────────
|
|
249
|
+
|
|
250
|
+
const listSecrets = os.listSecrets.handler(async () => {
|
|
251
|
+
const rows = await db.select().from(schema.secrets);
|
|
252
|
+
return rows.map((r) => ({
|
|
253
|
+
id: r.id,
|
|
254
|
+
name: r.name,
|
|
255
|
+
description: r.description,
|
|
256
|
+
createdAt: r.createdAt,
|
|
257
|
+
updatedAt: r.updatedAt,
|
|
258
|
+
}));
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
const createSecret = os.createSecret.handler(async ({ input }) => {
|
|
262
|
+
// Check for duplicate name
|
|
263
|
+
const existing = await db
|
|
264
|
+
.select()
|
|
265
|
+
.from(schema.secrets)
|
|
266
|
+
.where(eq(schema.secrets.name, input.name));
|
|
267
|
+
|
|
268
|
+
if (existing[0]) {
|
|
269
|
+
throw new ORPCError("CONFLICT", {
|
|
270
|
+
message: `Secret with name "${input.name}" already exists`,
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
const id = uuidv4();
|
|
275
|
+
const encryptedValue = encrypt(input.value);
|
|
276
|
+
await db.insert(schema.secrets).values({
|
|
277
|
+
id,
|
|
278
|
+
name: input.name,
|
|
279
|
+
encryptedValue,
|
|
280
|
+
description: input.description,
|
|
281
|
+
});
|
|
282
|
+
return { id, name: input.name };
|
|
283
|
+
});
|
|
284
|
+
|
|
285
|
+
const rotateSecret = os.rotateSecret.handler(async ({ input }) => {
|
|
286
|
+
const existing = await db
|
|
287
|
+
.select()
|
|
288
|
+
.from(schema.secrets)
|
|
289
|
+
.where(eq(schema.secrets.id, input.id));
|
|
290
|
+
|
|
291
|
+
if (!existing[0]) {
|
|
292
|
+
throw new ORPCError("NOT_FOUND", {
|
|
293
|
+
message: `Secret not found: ${input.id}`,
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
const encryptedValue = encrypt(input.value);
|
|
298
|
+
await db
|
|
299
|
+
.update(schema.secrets)
|
|
300
|
+
.set({ encryptedValue, updatedAt: new Date() })
|
|
301
|
+
.where(eq(schema.secrets.id, input.id));
|
|
302
|
+
|
|
303
|
+
return { success: true };
|
|
304
|
+
});
|
|
305
|
+
|
|
306
|
+
const deleteSecret = os.deleteSecret.handler(async ({ input }) => {
|
|
307
|
+
await db.delete(schema.secrets).where(eq(schema.secrets.id, input.id));
|
|
308
|
+
|
|
309
|
+
return { success: true };
|
|
310
|
+
});
|
|
311
|
+
|
|
312
|
+
const resolveSecret = os.resolveSecret.handler(async ({ input }) => {
|
|
313
|
+
const rows = await db
|
|
314
|
+
.select()
|
|
315
|
+
.from(schema.secrets)
|
|
316
|
+
.where(eq(schema.secrets.name, input.name));
|
|
317
|
+
|
|
318
|
+
const secret = rows[0];
|
|
319
|
+
if (!secret) {
|
|
320
|
+
throw new ORPCError("NOT_FOUND", {
|
|
321
|
+
message: `Secret not found: ${input.name}`,
|
|
322
|
+
});
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
return { value: decrypt(secret.encryptedValue) };
|
|
326
|
+
});
|
|
327
|
+
|
|
328
|
+
// ─── Kind Registry ────────────────────────────────────────────────────
|
|
329
|
+
|
|
330
|
+
const listKinds = os.listKinds.handler(async () => {
|
|
331
|
+
return kindRegistry.describeKinds();
|
|
332
|
+
});
|
|
333
|
+
|
|
334
|
+
// ─── Build Router ────────────────────────────────────────────────────
|
|
335
|
+
|
|
336
|
+
return os.router({
|
|
337
|
+
getProvenance,
|
|
338
|
+
listProvenance,
|
|
339
|
+
listProviders,
|
|
340
|
+
createProvider,
|
|
341
|
+
updateProvider,
|
|
342
|
+
deleteProvider,
|
|
343
|
+
triggerSync,
|
|
344
|
+
confirmOrphanDeletion,
|
|
345
|
+
dismissOrphan,
|
|
346
|
+
listSecrets,
|
|
347
|
+
createSecret,
|
|
348
|
+
rotateSecret,
|
|
349
|
+
deleteSecret,
|
|
350
|
+
resolveSecret,
|
|
351
|
+
listKinds,
|
|
352
|
+
});
|
|
353
|
+
};
|
|
354
|
+
|
|
355
|
+
export type GitOpsRouter = ReturnType<typeof createGitOpsRouter>;
|
package/src/schema.ts
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import {
|
|
2
|
+
pgTable,
|
|
3
|
+
pgEnum,
|
|
4
|
+
text,
|
|
5
|
+
timestamp,
|
|
6
|
+
integer,
|
|
7
|
+
} from "drizzle-orm/pg-core";
|
|
8
|
+
|
|
9
|
+
/** Enum for GitOps provider types. */
|
|
10
|
+
export const providerTypeEnum = pgEnum("provider_type", ["github", "gitlab"]);
|
|
11
|
+
|
|
12
|
+
/** Enum for deletion policy when descriptors disappear from git. */
|
|
13
|
+
export const deletionPolicyEnum = pgEnum("deletion_policy", [
|
|
14
|
+
"orphan",
|
|
15
|
+
"auto",
|
|
16
|
+
]);
|
|
17
|
+
|
|
18
|
+
/** Enum for provenance sync status. */
|
|
19
|
+
export const provenanceStatusEnum = pgEnum("provenance_status", [
|
|
20
|
+
"synced",
|
|
21
|
+
"error",
|
|
22
|
+
"orphaned",
|
|
23
|
+
]);
|
|
24
|
+
|
|
25
|
+
/** GitOps provider configurations (GitHub, GitLab, etc.) */
|
|
26
|
+
export const providers = pgTable("providers", {
|
|
27
|
+
id: text("id").primaryKey(),
|
|
28
|
+
type: providerTypeEnum("type").notNull(),
|
|
29
|
+
target: text("target").notNull(), // "my-org" or "my-org/my-repo"
|
|
30
|
+
pathPattern: text("path_pattern").notNull(), // e.g., ".checkstack/**/*.yaml"
|
|
31
|
+
/** Encrypted auth token (DynamicForm secret field). */
|
|
32
|
+
authToken: text("auth_token"),
|
|
33
|
+
/** Custom API base URL for enterprise/on-prem installations (e.g., "https://github.example.com/api/v3"). */
|
|
34
|
+
baseUrl: text("base_url"),
|
|
35
|
+
/** Sync interval in seconds. */
|
|
36
|
+
syncInterval: integer("sync_interval").notNull().default(300),
|
|
37
|
+
deletionPolicy: deletionPolicyEnum("deletion_policy")
|
|
38
|
+
.notNull()
|
|
39
|
+
.default("orphan"),
|
|
40
|
+
lastSyncAt: timestamp("last_sync_at"),
|
|
41
|
+
lastSyncError: text("last_sync_error"),
|
|
42
|
+
createdAt: timestamp("created_at").defaultNow().notNull(),
|
|
43
|
+
updatedAt: timestamp("updated_at").defaultNow().notNull(),
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
/** Provenance tracking for GitOps-managed entities. */
|
|
47
|
+
export const provenance = pgTable("provenance", {
|
|
48
|
+
id: text("id").primaryKey(),
|
|
49
|
+
apiVersion: text("api_version").notNull(),
|
|
50
|
+
kind: text("kind").notNull(),
|
|
51
|
+
entityName: text("entity_name").notNull(),
|
|
52
|
+
/** Plugin-specific entity ID (e.g., catalog system UUID). Set by the reconciler engine. */
|
|
53
|
+
entityId: text("entity_id").notNull(),
|
|
54
|
+
providerId: text("provider_id")
|
|
55
|
+
.notNull()
|
|
56
|
+
.references(() => providers.id, { onDelete: "cascade" }),
|
|
57
|
+
repository: text("repository").notNull(),
|
|
58
|
+
filePath: text("file_path").notNull(),
|
|
59
|
+
lastSyncHash: text("last_sync_hash").notNull(),
|
|
60
|
+
status: provenanceStatusEnum("status").notNull().default("synced"),
|
|
61
|
+
errorMessage: text("error_message"),
|
|
62
|
+
lastSyncedAt: timestamp("last_synced_at").defaultNow().notNull(),
|
|
63
|
+
createdAt: timestamp("created_at").defaultNow().notNull(),
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
/** Secret store for secretRef values in YAML descriptors. */
|
|
67
|
+
export const secrets = pgTable("secrets", {
|
|
68
|
+
id: text("id").primaryKey(),
|
|
69
|
+
/** Unique name referenced by secretRef in descriptors. */
|
|
70
|
+
name: text("name").notNull().unique(),
|
|
71
|
+
/** AES-256-GCM encrypted value (format: iv:authTag:ciphertext). */
|
|
72
|
+
encryptedValue: text("encrypted_value").notNull(),
|
|
73
|
+
description: text("description"),
|
|
74
|
+
createdBy: text("created_by"),
|
|
75
|
+
createdAt: timestamp("created_at").defaultNow().notNull(),
|
|
76
|
+
updatedAt: timestamp("updated_at").defaultNow().notNull(),
|
|
77
|
+
});
|