@secondlayer/shared 0.12.3 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -7
- package/dist/src/constants.d.ts +1 -1
- package/dist/src/constants.js.map +2 -2
- package/dist/src/db/index.d.ts +3 -64
- package/dist/src/db/index.js +2 -2
- package/dist/src/db/index.js.map +2 -2
- package/dist/src/db/queries/accounts.d.ts +2 -51
- package/dist/src/db/queries/integrity.d.ts +2 -51
- package/dist/src/db/queries/marketplace.d.ts +2 -51
- package/dist/src/db/queries/projects.d.ts +2 -51
- package/dist/src/db/queries/subgraph-gaps.d.ts +2 -51
- package/dist/src/db/queries/subgraphs.d.ts +3 -51
- package/dist/src/db/queries/subgraphs.js +3 -1
- package/dist/src/db/queries/subgraphs.js.map +3 -3
- package/dist/src/db/queries/usage.d.ts +3 -56
- package/dist/src/db/queries/usage.js +1 -19
- package/dist/src/db/queries/usage.js.map +4 -4
- package/dist/src/db/queries/workflows.d.ts +7 -53
- package/dist/src/db/queries/workflows.js +130 -13
- package/dist/src/db/queries/workflows.js.map +5 -4
- package/dist/src/db/schema.d.ts +3 -64
- package/dist/src/errors.d.ts +19 -50
- package/dist/src/errors.js +28 -45
- package/dist/src/errors.js.map +3 -3
- package/dist/src/index.d.ts +25 -256
- package/dist/src/index.js +32 -234
- package/dist/src/index.js.map +7 -9
- package/dist/src/lib/plans.d.ts +0 -1
- package/dist/src/lib/plans.js +1 -2
- package/dist/src/lib/plans.js.map +3 -3
- package/dist/src/node/local-client.d.ts +2 -51
- package/dist/src/queue/listener.d.ts +1 -18
- package/dist/src/queue/listener.js +2 -12
- package/dist/src/queue/listener.js.map +3 -3
- package/dist/src/schemas/filters.d.ts +3 -3
- package/dist/src/schemas/filters.js +3 -3
- package/dist/src/schemas/filters.js.map +3 -3
- package/dist/src/schemas/index.d.ts +5 -100
- package/dist/src/schemas/index.js +4 -88
- package/dist/src/schemas/index.js.map +5 -6
- package/dist/src/schemas/subgraphs.d.ts +2 -0
- package/dist/src/schemas/subgraphs.js +2 -1
- package/dist/src/schemas/subgraphs.js.map +3 -3
- package/dist/src/schemas/workflows.d.ts +4 -0
- package/dist/src/schemas/workflows.js +5 -1
- package/dist/src/schemas/workflows.js.map +3 -3
- package/dist/src/types.d.ts +1 -53
- package/migrations/0030_workflow_source_code.ts +21 -0
- package/migrations/0031_subgraph_source_code.ts +18 -0
- package/migrations/0032_drop_streams_tables.ts +43 -0
- package/package.json +2 -14
- package/dist/src/db/queries/metrics.d.ts +0 -419
- package/dist/src/db/queries/metrics.js +0 -55
- package/dist/src/db/queries/metrics.js.map +0 -10
- package/dist/src/queue/index.d.ts +0 -50
- package/dist/src/queue/index.js +0 -184
- package/dist/src/queue/index.js.map +0 -12
- package/dist/src/queue/recovery.d.ts +0 -14
- package/dist/src/queue/recovery.js +0 -108
- package/dist/src/queue/recovery.js.map +0 -12
package/dist/src/queue/index.js
DELETED
|
@@ -1,184 +0,0 @@
|
|
|
1
|
-
import { createRequire } from "node:module";
|
|
2
|
-
var __defProp = Object.defineProperty;
|
|
3
|
-
var __returnValue = (v) => v;
|
|
4
|
-
function __exportSetter(name, newValue) {
|
|
5
|
-
this[name] = __returnValue.bind(null, newValue);
|
|
6
|
-
}
|
|
7
|
-
var __export = (target, all) => {
|
|
8
|
-
for (var name in all)
|
|
9
|
-
__defProp(target, name, {
|
|
10
|
-
get: all[name],
|
|
11
|
-
enumerable: true,
|
|
12
|
-
configurable: true,
|
|
13
|
-
set: __exportSetter.bind(all, name)
|
|
14
|
-
});
|
|
15
|
-
};
|
|
16
|
-
|
|
17
|
-
// src/db/jsonb.ts
|
|
18
|
-
import { sql } from "kysely";
|
|
19
|
-
function jsonb(value) {
|
|
20
|
-
const escaped = JSON.stringify(value, (_k, v) => typeof v === "bigint" ? v.toString() : v).replace(/'/g, "''");
|
|
21
|
-
return sql`${sql.raw(`'${escaped}'::jsonb`)}`;
|
|
22
|
-
}
|
|
23
|
-
function parseJsonb(value) {
|
|
24
|
-
if (typeof value === "string") {
|
|
25
|
-
try {
|
|
26
|
-
return JSON.parse(value);
|
|
27
|
-
} catch {
|
|
28
|
-
return value;
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
return value ?? {};
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
// src/db/index.ts
|
|
35
|
-
import { Kysely } from "kysely";
|
|
36
|
-
import { PostgresJSDialect } from "kysely-postgres-js";
|
|
37
|
-
import postgres from "postgres";
|
|
38
|
-
import { sql as sql2 } from "kysely";
|
|
39
|
-
var db = null;
|
|
40
|
-
var rawClient = null;
|
|
41
|
-
function getDb(connectionString) {
|
|
42
|
-
if (!db) {
|
|
43
|
-
const url = connectionString || process.env.DATABASE_URL || "postgres://postgres:postgres@localhost:5432/streams_dev";
|
|
44
|
-
const isLocal = url.includes("localhost") || url.includes("127.0.0.1") || url.includes("@postgres:");
|
|
45
|
-
const poolMax = Number.parseInt(process.env.DATABASE_POOL_MAX ?? "20", 10);
|
|
46
|
-
rawClient = postgres(url, {
|
|
47
|
-
max: poolMax,
|
|
48
|
-
ssl: isLocal ? undefined : {
|
|
49
|
-
rejectUnauthorized: process.env.NODE_TLS_REJECT_UNAUTHORIZED !== "0"
|
|
50
|
-
}
|
|
51
|
-
});
|
|
52
|
-
db = new Kysely({
|
|
53
|
-
dialect: new PostgresJSDialect({ postgres: rawClient })
|
|
54
|
-
});
|
|
55
|
-
}
|
|
56
|
-
return db;
|
|
57
|
-
}
|
|
58
|
-
function getRawClient() {
|
|
59
|
-
if (!rawClient)
|
|
60
|
-
getDb();
|
|
61
|
-
return rawClient;
|
|
62
|
-
}
|
|
63
|
-
async function closeDb() {
|
|
64
|
-
if (db) {
|
|
65
|
-
await db.destroy();
|
|
66
|
-
db = null;
|
|
67
|
-
}
|
|
68
|
-
if (rawClient) {
|
|
69
|
-
await rawClient.end();
|
|
70
|
-
rawClient = null;
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
// src/queue/index.ts
|
|
74
|
-
var exports_queue = {};
|
|
75
|
-
__export(exports_queue, {
|
|
76
|
-
stats: () => stats,
|
|
77
|
-
getWorkerId: () => getWorkerId,
|
|
78
|
-
fail: () => fail,
|
|
79
|
-
enqueue: () => enqueue,
|
|
80
|
-
complete: () => complete,
|
|
81
|
-
claim: () => claim,
|
|
82
|
-
WORKER_ID: () => WORKER_ID
|
|
83
|
-
});
|
|
84
|
-
import { randomUUID } from "crypto";
|
|
85
|
-
import { sql as sql3 } from "kysely";
|
|
86
|
-
var WORKER_ID = `worker-${randomUUID().slice(0, 8)}`;
|
|
87
|
-
async function enqueue(streamId, blockHeight, backfill = false) {
|
|
88
|
-
const db2 = getDb();
|
|
89
|
-
const row = await db2.insertInto("jobs").values({
|
|
90
|
-
stream_id: streamId,
|
|
91
|
-
block_height: blockHeight,
|
|
92
|
-
backfill,
|
|
93
|
-
status: "pending",
|
|
94
|
-
attempts: 0
|
|
95
|
-
}).returning(["id"]).executeTakeFirstOrThrow();
|
|
96
|
-
return row.id;
|
|
97
|
-
}
|
|
98
|
-
async function claim() {
|
|
99
|
-
const db2 = getDb();
|
|
100
|
-
const { rows } = await sql3`
|
|
101
|
-
UPDATE jobs
|
|
102
|
-
SET
|
|
103
|
-
status = 'processing',
|
|
104
|
-
locked_at = NOW(),
|
|
105
|
-
locked_by = ${WORKER_ID},
|
|
106
|
-
attempts = attempts + 1
|
|
107
|
-
WHERE id = (
|
|
108
|
-
SELECT id FROM jobs
|
|
109
|
-
WHERE status = 'pending'
|
|
110
|
-
ORDER BY
|
|
111
|
-
backfill ASC,
|
|
112
|
-
block_height ASC,
|
|
113
|
-
created_at ASC
|
|
114
|
-
FOR UPDATE SKIP LOCKED
|
|
115
|
-
LIMIT 1
|
|
116
|
-
)
|
|
117
|
-
RETURNING *
|
|
118
|
-
`.execute(db2);
|
|
119
|
-
return rows[0] ?? null;
|
|
120
|
-
}
|
|
121
|
-
async function complete(jobId) {
|
|
122
|
-
const db2 = getDb();
|
|
123
|
-
await db2.updateTable("jobs").set({
|
|
124
|
-
status: "completed",
|
|
125
|
-
completed_at: new Date,
|
|
126
|
-
locked_at: null,
|
|
127
|
-
locked_by: null
|
|
128
|
-
}).where("id", "=", jobId).execute();
|
|
129
|
-
}
|
|
130
|
-
async function fail(jobId, error, maxAttempts = 3) {
|
|
131
|
-
const db2 = getDb();
|
|
132
|
-
const job = await db2.selectFrom("jobs").select("attempts").where("id", "=", jobId).executeTakeFirst();
|
|
133
|
-
if (!job)
|
|
134
|
-
return;
|
|
135
|
-
if (job.attempts < maxAttempts) {
|
|
136
|
-
await db2.updateTable("jobs").set({
|
|
137
|
-
status: "pending",
|
|
138
|
-
error,
|
|
139
|
-
locked_at: null,
|
|
140
|
-
locked_by: null
|
|
141
|
-
}).where("id", "=", jobId).execute();
|
|
142
|
-
} else {
|
|
143
|
-
await db2.updateTable("jobs").set({
|
|
144
|
-
status: "failed",
|
|
145
|
-
error,
|
|
146
|
-
completed_at: new Date,
|
|
147
|
-
locked_at: null,
|
|
148
|
-
locked_by: null
|
|
149
|
-
}).where("id", "=", jobId).execute();
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
async function stats() {
|
|
153
|
-
const { rows } = await sql3`
|
|
154
|
-
SELECT status, COUNT(*) as count
|
|
155
|
-
FROM jobs
|
|
156
|
-
GROUP BY status
|
|
157
|
-
`.execute(getDb());
|
|
158
|
-
const counts = {};
|
|
159
|
-
for (const row of rows) {
|
|
160
|
-
counts[row.status] = Number.parseInt(row.count, 10);
|
|
161
|
-
}
|
|
162
|
-
return {
|
|
163
|
-
pending: counts["pending"] || 0,
|
|
164
|
-
processing: counts["processing"] || 0,
|
|
165
|
-
completed: counts["completed"] || 0,
|
|
166
|
-
failed: counts["failed"] || 0,
|
|
167
|
-
total: Object.values(counts).reduce((a, b) => a + b, 0)
|
|
168
|
-
};
|
|
169
|
-
}
|
|
170
|
-
function getWorkerId() {
|
|
171
|
-
return WORKER_ID;
|
|
172
|
-
}
|
|
173
|
-
export {
|
|
174
|
-
stats,
|
|
175
|
-
getWorkerId,
|
|
176
|
-
fail,
|
|
177
|
-
enqueue,
|
|
178
|
-
complete,
|
|
179
|
-
claim,
|
|
180
|
-
WORKER_ID
|
|
181
|
-
};
|
|
182
|
-
|
|
183
|
-
//# debugId=1D149D7110903F9164756E2164756E21
|
|
184
|
-
//# sourceMappingURL=index.js.map
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../src/db/jsonb.ts", "../src/db/index.ts", "../src/queue/index.ts"],
|
|
4
|
-
"sourcesContent": [
|
|
5
|
-
"import { type RawBuilder, sql } from \"kysely\";\n\n/**\n * Safely encode a JS value as a JSONB literal for Kysely inserts/updates.\n * Kysely + postgres.js double-encodes JSON when using parameterized queries\n * with ::jsonb casts. This uses sql.raw to inline a properly escaped literal.\n */\nexport function jsonb(value: unknown): RawBuilder<unknown> {\n\tconst escaped = JSON.stringify(value, (_k, v) => (typeof v === \"bigint\" ? v.toString() : v)).replace(/'/g, \"''\");\n\treturn sql`${sql.raw(`'${escaped}'::jsonb`)}`;\n}\n\n/**\n * Safely parse a JSONB value from the database.\n * Handles double-encoded strings where postgres.js returns a JSON string\n * instead of a parsed object.\n */\nexport function parseJsonb<T = unknown>(value: unknown): T {\n\tif (typeof value === \"string\") {\n\t\ttry {\n\t\t\treturn JSON.parse(value) as T;\n\t\t} catch {\n\t\t\treturn value as T;\n\t\t}\n\t}\n\treturn (value ?? {}) as T;\n}\n",
|
|
6
|
-
"import { Kysely } from \"kysely\";\nimport { PostgresJSDialect } from \"kysely-postgres-js\";\nimport postgres from \"postgres\";\nimport type { Database } from \"./types.ts\";\n\nlet db: Kysely<Database> | null = null;\nlet rawClient: ReturnType<typeof postgres> | null = null;\n\nexport function getDb(connectionString?: string): Kysely<Database> {\n\tif (!db) {\n\t\tconst url =\n\t\t\tconnectionString ||\n\t\t\tprocess.env.DATABASE_URL ||\n\t\t\t\"postgres://postgres:postgres@localhost:5432/streams_dev\";\n\n\t\t// Always use SSL for remote databases, just disable cert verification if needed\n\t\tconst isLocal =\n\t\t\turl.includes(\"localhost\") ||\n\t\t\turl.includes(\"127.0.0.1\") ||\n\t\t\turl.includes(\"@postgres:\");\n\t\tconst poolMax = Number.parseInt(process.env.DATABASE_POOL_MAX ?? \"20\", 10);\n\t\trawClient = postgres(url, {\n\t\t\tmax: poolMax,\n\t\t\tssl: isLocal\n\t\t\t\t? undefined\n\t\t\t\t: {\n\t\t\t\t\t\trejectUnauthorized:\n\t\t\t\t\t\t\tprocess.env.NODE_TLS_REJECT_UNAUTHORIZED !== \"0\",\n\t\t\t\t\t},\n\t\t});\n\t\tdb = new Kysely<Database>({\n\t\t\tdialect: new PostgresJSDialect({ postgres: rawClient }),\n\t\t});\n\t}\n\treturn db;\n}\n\n/** Raw postgres.js client for dynamic schema DDL (CREATE SCHEMA, DROP, etc.) */\nexport function getRawClient(): ReturnType<typeof postgres> {\n\tif (!rawClient) getDb();\n\treturn rawClient!;\n}\n\n/** Close the DB connection pool. Call in CLI commands to allow process exit. */\nexport async function closeDb(): Promise<void> {\n\tif (db) {\n\t\tawait db.destroy();\n\t\tdb = null;\n\t}\n\tif (rawClient) {\n\t\tawait rawClient.end();\n\t\trawClient = null;\n\t}\n}\n\nimport { sql } from \"kysely\";\nexport { sql };\nexport * from \"./types.ts\";\nexport { jsonb, parseJsonb } from \"./jsonb.ts\";\n",
|
|
7
|
-
"import { randomUUID } from \"crypto\";\nimport { sql } from \"kysely\";\nimport { getDb } from \"../db/index.ts\";\nimport type { Job } from \"../db/types.ts\";\n\nexport interface QueueStats {\n\tpending: number;\n\tprocessing: number;\n\tcompleted: number;\n\tfailed: number;\n\ttotal: number;\n}\n\n// Worker identifier for this process\nconst WORKER_ID: string = `worker-${randomUUID().slice(0, 8)}`;\n\n/**\n * Enqueue a new job for stream evaluation\n */\nexport async function enqueue(\n\tstreamId: string,\n\tblockHeight: number,\n\tbackfill = false,\n): Promise<string> {\n\tconst db = getDb();\n\n\tconst row = await db\n\t\t.insertInto(\"jobs\")\n\t\t.values({\n\t\t\tstream_id: streamId,\n\t\t\tblock_height: blockHeight,\n\t\t\tbackfill,\n\t\t\tstatus: \"pending\",\n\t\t\tattempts: 0,\n\t\t})\n\t\t.returning([\"id\"])\n\t\t.executeTakeFirstOrThrow();\n\n\treturn row.id;\n}\n\n/**\n * Claim a pending job using SKIP LOCKED to prevent concurrent access\n * Returns null if no jobs available\n */\nexport async function claim(): Promise<Job | null> {\n\tconst db = getDb();\n\n\tconst { rows } = await sql<Job>`\n UPDATE jobs\n SET\n status = 'processing',\n locked_at = NOW(),\n locked_by = ${WORKER_ID},\n attempts = attempts + 1\n WHERE id = (\n SELECT id FROM jobs\n WHERE status = 'pending'\n ORDER BY\n backfill ASC,\n block_height ASC,\n created_at ASC\n FOR UPDATE SKIP LOCKED\n LIMIT 1\n )\n RETURNING *\n `.execute(db);\n\n\treturn rows[0] ?? null;\n}\n\n/**\n * Mark a job as completed\n */\nexport async function complete(jobId: string): Promise<void> {\n\tconst db = getDb();\n\n\tawait db\n\t\t.updateTable(\"jobs\")\n\t\t.set({\n\t\t\tstatus: \"completed\",\n\t\t\tcompleted_at: new Date(),\n\t\t\tlocked_at: null,\n\t\t\tlocked_by: null,\n\t\t})\n\t\t.where(\"id\", \"=\", jobId)\n\t\t.execute();\n}\n\n/**\n * Mark a job as failed\n * Re-queues if under max attempts, otherwise marks as permanently failed\n */\nexport async function fail(\n\tjobId: string,\n\terror: string,\n\tmaxAttempts = 3,\n): Promise<void> {\n\tconst db = getDb();\n\n\tconst job = await db\n\t\t.selectFrom(\"jobs\")\n\t\t.select(\"attempts\")\n\t\t.where(\"id\", \"=\", jobId)\n\t\t.executeTakeFirst();\n\n\tif (!job) return;\n\n\tif (job.attempts < maxAttempts) {\n\t\tawait db\n\t\t\t.updateTable(\"jobs\")\n\t\t\t.set({\n\t\t\t\tstatus: \"pending\",\n\t\t\t\terror,\n\t\t\t\tlocked_at: null,\n\t\t\t\tlocked_by: null,\n\t\t\t})\n\t\t\t.where(\"id\", \"=\", jobId)\n\t\t\t.execute();\n\t} else {\n\t\tawait db\n\t\t\t.updateTable(\"jobs\")\n\t\t\t.set({\n\t\t\t\tstatus: \"failed\",\n\t\t\t\terror,\n\t\t\t\tcompleted_at: new Date(),\n\t\t\t\tlocked_at: null,\n\t\t\t\tlocked_by: null,\n\t\t\t})\n\t\t\t.where(\"id\", \"=\", jobId)\n\t\t\t.execute();\n\t}\n}\n\n/**\n * Get queue statistics\n */\nexport async function stats(): Promise<QueueStats> {\n\tconst { rows } = await sql<{ status: string; count: string }>`\n SELECT status, COUNT(*) as count\n FROM jobs\n GROUP BY status\n `.execute(getDb());\n\n\tconst counts: Record<string, number> = {};\n\tfor (const row of rows) {\n\t\tcounts[row.status] = Number.parseInt(row.count, 10);\n\t}\n\n\treturn {\n\t\tpending: counts[\"pending\"] || 0,\n\t\tprocessing: counts[\"processing\"] || 0,\n\t\tcompleted: counts[\"completed\"] || 0,\n\t\tfailed: counts[\"failed\"] || 0,\n\t\ttotal: Object.values(counts).reduce((a, b) => a + b, 0),\n\t};\n}\n\n/**\n * Get worker ID for this process\n */\nexport function getWorkerId(): string {\n\treturn WORKER_ID;\n}\n\nexport { WORKER_ID };\n"
|
|
8
|
-
],
|
|
9
|
-
"mappings": ";;;;;;;;;;;;;;;;;AAAA;AAOO,SAAS,KAAK,CAAC,OAAqC;AAAA,EAC1D,MAAM,UAAU,KAAK,UAAU,OAAO,CAAC,IAAI,MAAO,OAAO,MAAM,WAAW,EAAE,SAAS,IAAI,CAAE,EAAE,QAAQ,MAAM,IAAI;AAAA,EAC/G,OAAO,MAAM,IAAI,IAAI,IAAI,iBAAiB;AAAA;AAQpC,SAAS,UAAuB,CAAC,OAAmB;AAAA,EAC1D,IAAI,OAAO,UAAU,UAAU;AAAA,IAC9B,IAAI;AAAA,MACH,OAAO,KAAK,MAAM,KAAK;AAAA,MACtB,MAAM;AAAA,MACP,OAAO;AAAA;AAAA,EAET;AAAA,EACA,OAAQ,SAAS,CAAC;AAAA;;;ACzBnB;AACA;AACA;AAqDA,gBAAS;AAlDT,IAAI,KAA8B;AAClC,IAAI,YAAgD;AAE7C,SAAS,KAAK,CAAC,kBAA6C;AAAA,EAClE,IAAI,CAAC,IAAI;AAAA,IACR,MAAM,MACL,oBACA,QAAQ,IAAI,gBACZ;AAAA,IAGD,MAAM,UACL,IAAI,SAAS,WAAW,KACxB,IAAI,SAAS,WAAW,KACxB,IAAI,SAAS,YAAY;AAAA,IAC1B,MAAM,UAAU,OAAO,SAAS,QAAQ,IAAI,qBAAqB,MAAM,EAAE;AAAA,IACzE,YAAY,SAAS,KAAK;AAAA,MACzB,KAAK;AAAA,MACL,KAAK,UACF,YACA;AAAA,QACA,oBACC,QAAQ,IAAI,iCAAiC;AAAA,MAC/C;AAAA,IACH,CAAC;AAAA,IACD,KAAK,IAAI,OAAiB;AAAA,MACzB,SAAS,IAAI,kBAAkB,EAAE,UAAU,UAAU,CAAC;AAAA,IACvD,CAAC;AAAA,EACF;AAAA,EACA,OAAO;AAAA;AAID,SAAS,YAAY,GAAgC;AAAA,EAC3D,IAAI,CAAC;AAAA,IAAW,MAAM;AAAA,EACtB,OAAO;AAAA;AAIR,eAAsB,OAAO,GAAkB;AAAA,EAC9C,IAAI,IAAI;AAAA,IACP,MAAM,GAAG,QAAQ;AAAA,IACjB,KAAK;AAAA,EACN;AAAA,EACA,IAAI,WAAW;AAAA,IACd,MAAM,UAAU,IAAI;AAAA,IACpB,YAAY;AAAA,EACb;AAAA;;;;;;;;;;;;ACpDD;AACA,gBAAS;AAaT,IAAM,YAAoB,UAAU,WAAW,EAAE,MAAM,GAAG,CAAC;AAK3D,eAAsB,OAAO,CAC5B,UACA,aACA,WAAW,OACO;AAAA,EAClB,MAAM,MAAK,MAAM;AAAA,EAEjB,MAAM,MAAM,MAAM,IAChB,WAAW,MAAM,EACjB,OAAO;AAAA,IACP,WAAW;AAAA,IACX,cAAc;AAAA,IACd;AAAA,IACA,QAAQ;AAAA,IACR,UAAU;AAAA,EACX,CAAC,EACA,UAAU,CAAC,IAAI,CAAC,EAChB,wBAAwB;AAAA,EAE1B,OAAO,IAAI;AAAA;AAOZ,eAAsB,KAAK,GAAwB;AAAA,EAClD,MAAM,MAAK,MAAM;AAAA,EAEjB,QAAQ,SAAS,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA,oBAKJ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAahB,QAAQ,GAAE;AAAA,EAEb,OAAO,KAAK,MAAM;AAAA;AAMnB,eAAsB,QAAQ,CAAC,OAA8B;AAAA,EAC5D,MAAM,MAAK,MAAM;AAAA,EAEjB,MAAM,IACJ,YAAY,MAAM,EAClB,IAAI;AAAA,IACJ,QAAQ;AAAA,IACR,cAAc,IAAI;AAAA,IAClB,WAAW;AAAA,IACX,WAAW;AAAA,EACZ,CAAC,EACA,MAAM,MAAM,KAAK,KAAK,EACtB,QAAQ;AAAA;AAOX,eAAsB,IAAI,CACzB,OACA,OACA,cAAc,GACE;AAAA,EAChB,MAAM,MAAK,MAAM;AAAA,EAEjB,MAAM,MAAM,MAAM,IAChB,WAAW,MAAM,EACjB,OAAO,UAAU,EACjB,MAAM,MAAM,KAAK,KAAK,EACtB,iBAAiB;AAAA,EAEnB,IAAI,CAAC;AAAA,IAAK;AAAA,EAEV,IAAI,IAAI,WAAW,aAAa;AAAA,IAC/B,MAAM,IACJ,YAAY,MAAM,EAClB,IAAI;AAAA,MACJ,QAAQ;AAAA,MACR;AAAA,MACA,WAAW;AAAA,MACX,WAAW;AAAA,IACZ,CAAC,EACA,MAAM,MAAM,KAAK,KAAK,EACtB,QAAQ;AAAA,EACX,EAAO;AAAA,IACN,MAAM,IACJ,YAAY,MAAM,EAClB,IAAI;AAAA,MACJ,QAAQ;AAAA,MACR;AAAA,MACA,cAAc,IAAI;AAAA,MAClB,WAAW;AAAA,MACX,WAAW;AAAA,IACZ,CAAC,EACA,MAAM,MAAM,KAAK,KAAK,EACtB,QAAQ;AAAA;AAAA;AAOZ,eAAsB,KAAK,GAAwB;AAAA,EAClD,QAAQ,SAAS,MAAM;AAAA;AAAA;AAAA;AAAA,IAIpB,QAAQ,MAAM,CAAC;AAAA,EAElB,MAAM,SAAiC,CAAC;AAAA,EACxC,WAAW,OAAO,MAAM;AAAA,IACvB,OAAO,IAAI,UAAU,OAAO,SAAS,IAAI,OAAO,EAAE;AAAA,EACnD;AAAA,EAEA,OAAO;AAAA,IACN,SAAS,OAAO,cAAc;AAAA,IAC9B,YAAY,OAAO,iBAAiB;AAAA,IACpC,WAAW,OAAO,gBAAgB;AAAA,IAClC,QAAQ,OAAO,aAAa;AAAA,IAC5B,OAAO,OAAO,OAAO,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC;AAAA,EACvD;AAAA;AAMM,SAAS,WAAW,GAAW;AAAA,EACrC,OAAO;AAAA;",
|
|
10
|
-
"debugId": "1D149D7110903F9164756E2164756E21",
|
|
11
|
-
"names": []
|
|
12
|
-
}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Recover jobs that have been locked for longer than the threshold
|
|
3
|
-
* These are likely from crashed workers
|
|
4
|
-
*
|
|
5
|
-
* @param staleThresholdMinutes - Minutes after which a locked job is considered stale
|
|
6
|
-
* @returns Number of recovered jobs
|
|
7
|
-
*/
|
|
8
|
-
declare function recoverStaleJobs(staleThresholdMinutes?: number): Promise<number>;
|
|
9
|
-
/**
|
|
10
|
-
* Run periodic stale job recovery
|
|
11
|
-
* Returns a cleanup function to stop the interval
|
|
12
|
-
*/
|
|
13
|
-
declare function startRecoveryLoop(intervalMs?: number, staleThresholdMinutes?: number): () => void;
|
|
14
|
-
export { startRecoveryLoop, recoverStaleJobs };
|
|
@@ -1,108 +0,0 @@
|
|
|
1
|
-
import { createRequire } from "node:module";
|
|
2
|
-
var __defProp = Object.defineProperty;
|
|
3
|
-
var __returnValue = (v) => v;
|
|
4
|
-
function __exportSetter(name, newValue) {
|
|
5
|
-
this[name] = __returnValue.bind(null, newValue);
|
|
6
|
-
}
|
|
7
|
-
var __export = (target, all) => {
|
|
8
|
-
for (var name in all)
|
|
9
|
-
__defProp(target, name, {
|
|
10
|
-
get: all[name],
|
|
11
|
-
enumerable: true,
|
|
12
|
-
configurable: true,
|
|
13
|
-
set: __exportSetter.bind(all, name)
|
|
14
|
-
});
|
|
15
|
-
};
|
|
16
|
-
|
|
17
|
-
// src/db/jsonb.ts
|
|
18
|
-
import { sql } from "kysely";
|
|
19
|
-
function jsonb(value) {
|
|
20
|
-
const escaped = JSON.stringify(value, (_k, v) => typeof v === "bigint" ? v.toString() : v).replace(/'/g, "''");
|
|
21
|
-
return sql`${sql.raw(`'${escaped}'::jsonb`)}`;
|
|
22
|
-
}
|
|
23
|
-
function parseJsonb(value) {
|
|
24
|
-
if (typeof value === "string") {
|
|
25
|
-
try {
|
|
26
|
-
return JSON.parse(value);
|
|
27
|
-
} catch {
|
|
28
|
-
return value;
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
return value ?? {};
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
// src/db/index.ts
|
|
35
|
-
import { Kysely } from "kysely";
|
|
36
|
-
import { PostgresJSDialect } from "kysely-postgres-js";
|
|
37
|
-
import postgres from "postgres";
|
|
38
|
-
import { sql as sql2 } from "kysely";
|
|
39
|
-
var db = null;
|
|
40
|
-
var rawClient = null;
|
|
41
|
-
function getDb(connectionString) {
|
|
42
|
-
if (!db) {
|
|
43
|
-
const url = connectionString || process.env.DATABASE_URL || "postgres://postgres:postgres@localhost:5432/streams_dev";
|
|
44
|
-
const isLocal = url.includes("localhost") || url.includes("127.0.0.1") || url.includes("@postgres:");
|
|
45
|
-
const poolMax = Number.parseInt(process.env.DATABASE_POOL_MAX ?? "20", 10);
|
|
46
|
-
rawClient = postgres(url, {
|
|
47
|
-
max: poolMax,
|
|
48
|
-
ssl: isLocal ? undefined : {
|
|
49
|
-
rejectUnauthorized: process.env.NODE_TLS_REJECT_UNAUTHORIZED !== "0"
|
|
50
|
-
}
|
|
51
|
-
});
|
|
52
|
-
db = new Kysely({
|
|
53
|
-
dialect: new PostgresJSDialect({ postgres: rawClient })
|
|
54
|
-
});
|
|
55
|
-
}
|
|
56
|
-
return db;
|
|
57
|
-
}
|
|
58
|
-
function getRawClient() {
|
|
59
|
-
if (!rawClient)
|
|
60
|
-
getDb();
|
|
61
|
-
return rawClient;
|
|
62
|
-
}
|
|
63
|
-
async function closeDb() {
|
|
64
|
-
if (db) {
|
|
65
|
-
await db.destroy();
|
|
66
|
-
db = null;
|
|
67
|
-
}
|
|
68
|
-
if (rawClient) {
|
|
69
|
-
await rawClient.end();
|
|
70
|
-
rawClient = null;
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
// src/queue/recovery.ts
|
|
74
|
-
import { sql as sql3 } from "kysely";
|
|
75
|
-
async function recoverStaleJobs(staleThresholdMinutes = 5) {
|
|
76
|
-
const { rows } = await sql3`
|
|
77
|
-
UPDATE jobs
|
|
78
|
-
SET
|
|
79
|
-
status = 'pending',
|
|
80
|
-
locked_at = NULL,
|
|
81
|
-
locked_by = NULL
|
|
82
|
-
WHERE
|
|
83
|
-
status = 'processing'
|
|
84
|
-
AND locked_at < NOW() - INTERVAL '${sql3.raw(staleThresholdMinutes.toString())} minutes'
|
|
85
|
-
RETURNING id
|
|
86
|
-
`.execute(getDb());
|
|
87
|
-
return rows.length;
|
|
88
|
-
}
|
|
89
|
-
function startRecoveryLoop(intervalMs = 60000, staleThresholdMinutes = 5) {
|
|
90
|
-
const intervalId = setInterval(async () => {
|
|
91
|
-
try {
|
|
92
|
-
const recovered = await recoverStaleJobs(staleThresholdMinutes);
|
|
93
|
-
if (recovered > 0) {
|
|
94
|
-
console.log(`Recovered ${recovered} stale jobs`);
|
|
95
|
-
}
|
|
96
|
-
} catch (error) {
|
|
97
|
-
console.error("Error recovering stale jobs:", error);
|
|
98
|
-
}
|
|
99
|
-
}, intervalMs);
|
|
100
|
-
return () => clearInterval(intervalId);
|
|
101
|
-
}
|
|
102
|
-
export {
|
|
103
|
-
startRecoveryLoop,
|
|
104
|
-
recoverStaleJobs
|
|
105
|
-
};
|
|
106
|
-
|
|
107
|
-
//# debugId=C42C2C4D6C2BBDFE64756E2164756E21
|
|
108
|
-
//# sourceMappingURL=recovery.js.map
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../src/db/jsonb.ts", "../src/db/index.ts", "../src/queue/recovery.ts"],
|
|
4
|
-
"sourcesContent": [
|
|
5
|
-
"import { type RawBuilder, sql } from \"kysely\";\n\n/**\n * Safely encode a JS value as a JSONB literal for Kysely inserts/updates.\n * Kysely + postgres.js double-encodes JSON when using parameterized queries\n * with ::jsonb casts. This uses sql.raw to inline a properly escaped literal.\n */\nexport function jsonb(value: unknown): RawBuilder<unknown> {\n\tconst escaped = JSON.stringify(value, (_k, v) => (typeof v === \"bigint\" ? v.toString() : v)).replace(/'/g, \"''\");\n\treturn sql`${sql.raw(`'${escaped}'::jsonb`)}`;\n}\n\n/**\n * Safely parse a JSONB value from the database.\n * Handles double-encoded strings where postgres.js returns a JSON string\n * instead of a parsed object.\n */\nexport function parseJsonb<T = unknown>(value: unknown): T {\n\tif (typeof value === \"string\") {\n\t\ttry {\n\t\t\treturn JSON.parse(value) as T;\n\t\t} catch {\n\t\t\treturn value as T;\n\t\t}\n\t}\n\treturn (value ?? {}) as T;\n}\n",
|
|
6
|
-
"import { Kysely } from \"kysely\";\nimport { PostgresJSDialect } from \"kysely-postgres-js\";\nimport postgres from \"postgres\";\nimport type { Database } from \"./types.ts\";\n\nlet db: Kysely<Database> | null = null;\nlet rawClient: ReturnType<typeof postgres> | null = null;\n\nexport function getDb(connectionString?: string): Kysely<Database> {\n\tif (!db) {\n\t\tconst url =\n\t\t\tconnectionString ||\n\t\t\tprocess.env.DATABASE_URL ||\n\t\t\t\"postgres://postgres:postgres@localhost:5432/streams_dev\";\n\n\t\t// Always use SSL for remote databases, just disable cert verification if needed\n\t\tconst isLocal =\n\t\t\turl.includes(\"localhost\") ||\n\t\t\turl.includes(\"127.0.0.1\") ||\n\t\t\turl.includes(\"@postgres:\");\n\t\tconst poolMax = Number.parseInt(process.env.DATABASE_POOL_MAX ?? \"20\", 10);\n\t\trawClient = postgres(url, {\n\t\t\tmax: poolMax,\n\t\t\tssl: isLocal\n\t\t\t\t? undefined\n\t\t\t\t: {\n\t\t\t\t\t\trejectUnauthorized:\n\t\t\t\t\t\t\tprocess.env.NODE_TLS_REJECT_UNAUTHORIZED !== \"0\",\n\t\t\t\t\t},\n\t\t});\n\t\tdb = new Kysely<Database>({\n\t\t\tdialect: new PostgresJSDialect({ postgres: rawClient }),\n\t\t});\n\t}\n\treturn db;\n}\n\n/** Raw postgres.js client for dynamic schema DDL (CREATE SCHEMA, DROP, etc.) */\nexport function getRawClient(): ReturnType<typeof postgres> {\n\tif (!rawClient) getDb();\n\treturn rawClient!;\n}\n\n/** Close the DB connection pool. Call in CLI commands to allow process exit. */\nexport async function closeDb(): Promise<void> {\n\tif (db) {\n\t\tawait db.destroy();\n\t\tdb = null;\n\t}\n\tif (rawClient) {\n\t\tawait rawClient.end();\n\t\trawClient = null;\n\t}\n}\n\nimport { sql } from \"kysely\";\nexport { sql };\nexport * from \"./types.ts\";\nexport { jsonb, parseJsonb } from \"./jsonb.ts\";\n",
|
|
7
|
-
"import { sql } from \"kysely\";\nimport { getDb } from \"../db/index.ts\";\n\n/**\n * Recover jobs that have been locked for longer than the threshold\n * These are likely from crashed workers\n *\n * @param staleThresholdMinutes - Minutes after which a locked job is considered stale\n * @returns Number of recovered jobs\n */\nexport async function recoverStaleJobs(\n\tstaleThresholdMinutes = 5,\n): Promise<number> {\n\tconst { rows } = await sql`\n UPDATE jobs\n SET\n status = 'pending',\n locked_at = NULL,\n locked_by = NULL\n WHERE\n status = 'processing'\n AND locked_at < NOW() - INTERVAL '${sql.raw(staleThresholdMinutes.toString())} minutes'\n RETURNING id\n `.execute(getDb());\n\n\treturn rows.length;\n}\n\n/**\n * Run periodic stale job recovery\n * Returns a cleanup function to stop the interval\n */\nexport function startRecoveryLoop(\n\tintervalMs = 60000, // 1 minute\n\tstaleThresholdMinutes = 5,\n): () => void {\n\tconst intervalId = setInterval(async () => {\n\t\ttry {\n\t\t\tconst recovered = await recoverStaleJobs(staleThresholdMinutes);\n\t\t\tif (recovered > 0) {\n\t\t\t\tconsole.log(`Recovered ${recovered} stale jobs`);\n\t\t\t}\n\t\t} catch (error) {\n\t\t\tconsole.error(\"Error recovering stale jobs:\", error);\n\t\t}\n\t}, intervalMs);\n\n\treturn () => clearInterval(intervalId);\n}\n"
|
|
8
|
-
],
|
|
9
|
-
"mappings": ";;;;;;;;;;;;;;;;;AAAA;AAOO,SAAS,KAAK,CAAC,OAAqC;AAAA,EAC1D,MAAM,UAAU,KAAK,UAAU,OAAO,CAAC,IAAI,MAAO,OAAO,MAAM,WAAW,EAAE,SAAS,IAAI,CAAE,EAAE,QAAQ,MAAM,IAAI;AAAA,EAC/G,OAAO,MAAM,IAAI,IAAI,IAAI,iBAAiB;AAAA;AAQpC,SAAS,UAAuB,CAAC,OAAmB;AAAA,EAC1D,IAAI,OAAO,UAAU,UAAU;AAAA,IAC9B,IAAI;AAAA,MACH,OAAO,KAAK,MAAM,KAAK;AAAA,MACtB,MAAM;AAAA,MACP,OAAO;AAAA;AAAA,EAET;AAAA,EACA,OAAQ,SAAS,CAAC;AAAA;;;ACzBnB;AACA;AACA;AAqDA,gBAAS;AAlDT,IAAI,KAA8B;AAClC,IAAI,YAAgD;AAE7C,SAAS,KAAK,CAAC,kBAA6C;AAAA,EAClE,IAAI,CAAC,IAAI;AAAA,IACR,MAAM,MACL,oBACA,QAAQ,IAAI,gBACZ;AAAA,IAGD,MAAM,UACL,IAAI,SAAS,WAAW,KACxB,IAAI,SAAS,WAAW,KACxB,IAAI,SAAS,YAAY;AAAA,IAC1B,MAAM,UAAU,OAAO,SAAS,QAAQ,IAAI,qBAAqB,MAAM,EAAE;AAAA,IACzE,YAAY,SAAS,KAAK;AAAA,MACzB,KAAK;AAAA,MACL,KAAK,UACF,YACA;AAAA,QACA,oBACC,QAAQ,IAAI,iCAAiC;AAAA,MAC/C;AAAA,IACH,CAAC;AAAA,IACD,KAAK,IAAI,OAAiB;AAAA,MACzB,SAAS,IAAI,kBAAkB,EAAE,UAAU,UAAU,CAAC;AAAA,IACvD,CAAC;AAAA,EACF;AAAA,EACA,OAAO;AAAA;AAID,SAAS,YAAY,GAAgC;AAAA,EAC3D,IAAI,CAAC;AAAA,IAAW,MAAM;AAAA,EACtB,OAAO;AAAA;AAIR,eAAsB,OAAO,GAAkB;AAAA,EAC9C,IAAI,IAAI;AAAA,IACP,MAAM,GAAG,QAAQ;AAAA,IACjB,KAAK;AAAA,EACN;AAAA,EACA,IAAI,WAAW;AAAA,IACd,MAAM,UAAU,IAAI;AAAA,IACpB,YAAY;AAAA,EACb;AAAA;;ACpDD,gBAAS;AAUT,eAAsB,gBAAgB,CACrC,wBAAwB,GACN;AAAA,EAClB,QAAQ,SAAS,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,0CAQkB,KAAI,IAAI,sBAAsB,SAAS,CAAC;AAAA;AAAA,IAE9E,QAAQ,MAAM,CAAC;AAAA,EAElB,OAAO,KAAK;AAAA;AAON,SAAS,iBAAiB,CAChC,aAAa,OACb,wBAAwB,GACX;AAAA,EACb,MAAM,aAAa,YAAY,YAAY;AAAA,IAC1C,IAAI;AAAA,MACH,MAAM,YAAY,MAAM,iBAAiB,qBAAqB;AAAA,MAC9D,IAAI,YAAY,GAAG;AAAA,QAClB,QAAQ,IAAI,aAAa,sBAAsB;AAAA,MAChD;AAAA,MACC,OAAO,OAAO;AAAA,MACf,QAAQ,MAAM,gCAAgC,KAAK;AAAA;AAAA,KAElD,UAAU;AAAA,EAEb,OAAO,MAAM,cAAc,UAAU;AAAA;",
|
|
10
|
-
"debugId": "C42C2C4D6C2BBDFE64756E2164756E21",
|
|
11
|
-
"names": []
|
|
12
|
-
}
|