@ventually/sqlite 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.d.ts ADDED
@@ -0,0 +1,7 @@
1
+ #!/usr/bin/env node
2
+ import { type SQLiteAdapterOptions } from "./index.js";
3
+ declare function main(argv?: string[]): Promise<void>;
4
+ declare function readAdapterOptions(args: string[]): SQLiteAdapterOptions;
5
+ declare function readFlagValue(args: string[], flag: string): string | null;
6
+ export { main, readAdapterOptions, readFlagValue };
7
+ //# sourceMappingURL=cli.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAIA,OAAO,EAAiB,KAAK,oBAAoB,EAAE,MAAM,YAAY,CAAC;AA4CtE,iBAAe,IAAI,CAAC,IAAI,WAAwB,GAAG,OAAO,CAAC,IAAI,CAAC,CAU/D;AAED,iBAAS,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,oBAAoB,CAwBhE;AAED,iBAAS,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CA0BlE;AA2CD,OAAO,EAAE,IAAI,EAAE,kBAAkB,EAAE,aAAa,EAAE,CAAC"}
package/dist/cli.js ADDED
@@ -0,0 +1,121 @@
1
+ #!/usr/bin/env node
2
+ import process from "node:process";
3
+ import { SQLiteAdapter } from "./index.js";
4
+ const migrateDescription = "Create the adapter tables and indexes if they do not exist.";
5
+ const helpDescription = "Show available commands.";
6
+ const versionDescription = "Print the installed package version.";
7
+ const commands = {
8
+ help: {
9
+ description: helpDescription,
10
+ run: async () => {
11
+ printHelp();
12
+ },
13
+ },
14
+ migrate: {
15
+ description: migrateDescription,
16
+ run: async (args) => {
17
+ const options = readAdapterOptions(args);
18
+ const adapter = new SQLiteAdapter(options);
19
+ try {
20
+ await adapter.migrate();
21
+ }
22
+ finally {
23
+ await adapter.close();
24
+ }
25
+ console.log(`Migrated @ventually/sqlite schema for ${describeConnection(options.connection)}${options.namespace ? ` (namespace: ${options.namespace})` : ""}.`);
26
+ },
27
+ },
28
+ version: {
29
+ description: versionDescription,
30
+ run: async () => {
31
+ console.log(process.env.npm_package_version ?? "0.0.0");
32
+ },
33
+ },
34
+ };
35
+ async function main(argv = process.argv.slice(2)) {
36
+ const [rawCommand, ...rest] = argv;
37
+ const commandName = rawCommand ?? "help";
38
+ if (!Object.hasOwn(commands, commandName)) {
39
+ throw new Error(`Unsupported command: ${commandName}`);
40
+ }
41
+ const command = commands[commandName];
42
+ await command.run(rest);
43
+ }
44
+ function readAdapterOptions(args) {
45
+ const url = readFlagValue(args, "--connection") ??
46
+ readFlagValue(args, "--url") ??
47
+ process.env.EVENTUALLY_SQLITE_URL ??
48
+ process.env.DATABASE_URL ??
49
+ process.env.TURSO_DATABASE_URL;
50
+ if (!url) {
51
+ throw new Error("Missing SQLite connection. Pass --connection <url> or set EVENTUALLY_SQLITE_URL/DATABASE_URL.");
52
+ }
53
+ const authToken = readFlagValue(args, "--auth-token") ??
54
+ process.env.EVENTUALLY_SQLITE_AUTH_TOKEN ??
55
+ process.env.TURSO_AUTH_TOKEN;
56
+ const namespace = readFlagValue(args, "--namespace") ?? process.env.EVENTUALLY_SQLITE_NAMESPACE;
57
+ return {
58
+ connection: authToken ? { url, authToken } : url,
59
+ namespace: namespace || undefined,
60
+ };
61
+ }
62
+ function readFlagValue(args, flag) {
63
+ const equalsPrefix = `${flag}=`;
64
+ for (let index = 0; index < args.length; index += 1) {
65
+ const arg = args[index];
66
+ if (!arg) {
67
+ continue;
68
+ }
69
+ if (arg === flag) {
70
+ const value = args[index + 1];
71
+ if (!value || value.startsWith("-")) {
72
+ throw new Error(`Missing value for ${flag}.`);
73
+ }
74
+ return value;
75
+ }
76
+ if (arg.startsWith(equalsPrefix)) {
77
+ const value = arg.slice(equalsPrefix.length);
78
+ if (!value) {
79
+ throw new Error(`Missing value for ${flag}.`);
80
+ }
81
+ return value;
82
+ }
83
+ }
84
+ return null;
85
+ }
86
+ function describeConnection(connection) {
87
+ if (typeof connection === "string") {
88
+ return connection;
89
+ }
90
+ return connection?.url ?? ":memory:";
91
+ }
92
+ function printHelp() {
93
+ console.log(`Usage: bunx @ventually/sqlite <command> [options]
94
+
95
+ Commands:
96
+ migrate ${migrateDescription}
97
+ help ${helpDescription}
98
+ version ${versionDescription}
99
+
100
+ Options for migrate:
101
+ --connection, --url <url> SQLite/libSQL connection string
102
+ --auth-token <token> Optional auth token for remote libSQL
103
+ --namespace <name> Optional adapter namespace
104
+
105
+ Env fallbacks:
106
+ EVENTUALLY_SQLITE_URL
107
+ EVENTUALLY_SQLITE_AUTH_TOKEN
108
+ EVENTUALLY_SQLITE_NAMESPACE
109
+ DATABASE_URL
110
+ TURSO_DATABASE_URL
111
+ TURSO_AUTH_TOKEN`);
112
+ }
113
+ const isDirectExecution = process.argv[1] !== undefined && import.meta.url === new URL(`file://${process.argv[1]}`).href;
114
+ if (isDirectExecution) {
115
+ void main().catch((error) => {
116
+ const message = error instanceof Error ? error.message : String(error);
117
+ console.error(message);
118
+ process.exit(1);
119
+ });
120
+ }
121
+ export { main, readAdapterOptions, readFlagValue };
@@ -0,0 +1,3 @@
1
+ export { SQLiteAdapter } from "./sqlite-adapter.js";
2
+ export type { SQLiteAdapterOptions } from "./sqlite-adapter.js";
3
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,YAAY,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC"}
package/dist/index.js ADDED
@@ -0,0 +1 @@
1
+ export { SQLiteAdapter } from "./sqlite-adapter.js";
@@ -0,0 +1,56 @@
1
+ import { type Config } from "@libsql/client";
2
+ import type { AdapterClaimOptions, AdapterEnqueueRequest, AdapterFailureRecord, AdapterJobRecord, IQueueAdapter } from "@ventually/core";
3
+ type KeepMode = {
4
+ mode: "all" | "none" | "count";
5
+ count?: number;
6
+ };
7
+ export interface SQLiteAdapterOptions {
8
+ connection?: string | Config;
9
+ namespace?: string;
10
+ }
11
+ export declare class SQLiteAdapter implements IQueueAdapter {
12
+ private readonly client;
13
+ private readonly db;
14
+ private readonly namespace;
15
+ private schemaPromise;
16
+ private closed;
17
+ constructor(options?: SQLiteAdapterOptions);
18
+ enqueue(request: AdapterEnqueueRequest): Promise<AdapterJobRecord>;
19
+ enqueueMany(requests: AdapterEnqueueRequest[]): Promise<AdapterJobRecord[]>;
20
+ getJob(queueName: string, jobId: string): Promise<AdapterJobRecord | null>;
21
+ claimNext(queueName: string, options: AdapterClaimOptions): Promise<AdapterJobRecord[]>;
22
+ complete(queueName: string, jobId: string, lockToken: string, result: unknown, finishedAt: number, keep: KeepMode): Promise<void>;
23
+ fail(queueName: string, jobId: string, lockToken: string, failure: AdapterFailureRecord, finishedAt: number, retryAt: number | null, keep: KeepMode): Promise<void>;
24
+ updateProgress(queueName: string, jobId: string, lockToken: string, progress: unknown, updatedAt: number): Promise<void>;
25
+ recoverStalled(queueName: string, now: number, lockTtlMs: number, limit: number): Promise<AdapterJobRecord[]>;
26
+ migrate(): Promise<void>;
27
+ close(): Promise<void>;
28
+ private ensureReady;
29
+ private createSchema;
30
+ private allocateSequences;
31
+ private createJobRow;
32
+ private toRecord;
33
+ private findJob;
34
+ private requireJob;
35
+ private assertLock;
36
+ private createLockToken;
37
+ private writeJob;
38
+ private trimTerminal;
39
+ private resolveParent;
40
+ private computeRetryAt;
41
+ private parseParentKey;
42
+ private matchJob;
43
+ private jobKey;
44
+ private encode;
45
+ private decode;
46
+ private decodeBackoff;
47
+ private decodeKeep;
48
+ private withTransaction;
49
+ private normalizeConnection;
50
+ private normalizeUrl;
51
+ private createInMemoryUrl;
52
+ private isRetryableSqliteError;
53
+ private sleep;
54
+ }
55
+ export {};
56
+ //# sourceMappingURL=sqlite-adapter.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"sqlite-adapter.d.ts","sourceRoot":"","sources":["../src/sqlite-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAA6B,KAAK,MAAM,EAAE,MAAM,gBAAgB,CAAC;AASxE,OAAO,KAAK,EACV,mBAAmB,EACnB,qBAAqB,EACrB,oBAAoB,EACpB,gBAAgB,EAChB,aAAa,EACd,MAAM,iBAAiB,CAAC;AA2EzB,KAAK,QAAQ,GAAG;IAAE,IAAI,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAInE,MAAM,WAAW,oBAAoB;IACnC,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,aAAc,YAAW,aAAa;IACjD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAyB;IAC5C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,aAAa,CAA8B;IACnD,OAAO,CAAC,MAAM,CAAS;gBAEX,OAAO,GAAE,oBAAyB;IAMxC,OAAO,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAoBlE,WAAW,CAAC,QAAQ,EAAE,qBAAqB,EAAE,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAqE3E,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAM1E,SAAS,CACb,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,mBAAmB,GAC3B,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAmDxB,QAAQ,CACZ,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,OAAO,EACf,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,QAAQ,GACb,OAAO,CAAC,IAAI,CAAC;IA2BV,IAAI,CACR,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,oBAAoB,EAC7B,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,GAAG,IAAI,EACtB,IAAI,EAAE,QAAQ,GACb,OAAO,CAAC,IAAI,CAAC;IA4CV,cAAc,CAClB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,OAAO,EACjB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC;IAiBV,cAAc,CAClB,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,gBAAgB,EAAE,CAAC;IA6DxB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAIxB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAQd,WAAW;YAUX,YAAY;YAwDZ,iBAAiB;IAmC/B,OAAO,CAAC,YAAY;IAsCpB,OAAO,CAAC,QAAQ;YA4BF,OAAO;YAUP,UAAU;IAQxB,OAAO,CAAC,UAAU;IAMlB,OAAO,CAAC,eAAe;YAIT,QAAQ;YAwBR,YAAY;YA8CZ,aAAa;IA+D3B,OAAO,CAAC,cAAc;IActB,OAAO,CAAC,cAAc;IAYtB,OAAO,CAAC,QAAQ;IAQhB,OAAO,CAAC,MAAM;IAId,OAAO,CAAC,MAAM;IAId,OAAO,CAAC,MAAM;IAId,OAAO,CAAC,aAAa;IAIrB,OAAO,CAAC,UAAU;YAIJ,eAAe;IAiB7B,OAAO,CAAC,mBAAmB;IAe3B,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,iBAAiB;IAIzB,OAAO,CAAC,sBAAsB;IAS9B,OAAO,CAAC,KAAK;CAGd"}
@@ -0,0 +1,648 @@
1
+ import { createClient } from "@libsql/client";
2
+ import { and, asc, count, desc, eq, inArray, lte, or, sql } from "drizzle-orm";
3
+ import { drizzle, } from "drizzle-orm/libsql";
4
+ import { index, integer, primaryKey, sqliteTable, text } from "drizzle-orm/sqlite-core";
5
+ const SEQUENCE_KEY = "sequence";
6
+ const TRANSACTION_MAX_RETRIES = 32;
7
+ const counters = sqliteTable("eventually_counters", {
8
+ namespace: text("namespace").notNull(),
9
+ key: text("key").notNull(),
10
+ value: integer("value").notNull(),
11
+ }, (table) => ({
12
+ pk: primaryKey({ columns: [table.namespace, table.key] }),
13
+ }));
14
+ const jobs = sqliteTable("eventually_jobs", {
15
+ namespace: text("namespace").notNull(),
16
+ queueName: text("queue_name").notNull(),
17
+ jobId: text("job_id").notNull(),
18
+ name: text("name").notNull(),
19
+ data: text("data").notNull(),
20
+ state: text("state").notNull().$type(),
21
+ attempt: integer("attempt").notNull(),
22
+ attempts: integer("attempts").notNull(),
23
+ priority: integer("priority").notNull(),
24
+ delay: integer("delay").notNull(),
25
+ createdAt: integer("created_at").notNull(),
26
+ processedAt: integer("processed_at"),
27
+ finishedAt: integer("finished_at"),
28
+ result: text("result"),
29
+ failedReason: text("failed_reason"),
30
+ stack: text("stack"),
31
+ progress: text("progress"),
32
+ lockToken: text("lock_token"),
33
+ availableAt: integer("available_at").notNull(),
34
+ backoff: text("backoff"),
35
+ removeOnComplete: text("remove_on_complete").notNull(),
36
+ removeOnFail: text("remove_on_fail").notNull(),
37
+ parentKey: text("parent_key"),
38
+ pendingChildren: integer("pending_children").notNull(),
39
+ childFailure: text("child_failure"),
40
+ sequence: integer("sequence").notNull(),
41
+ }, (table) => ({
42
+ pk: primaryKey({ columns: [table.namespace, table.queueName, table.jobId] }),
43
+ waitingIdx: index("eventually_jobs_waiting_idx").on(table.namespace, table.queueName, table.state, table.priority, table.sequence),
44
+ availableIdx: index("eventually_jobs_available_idx").on(table.namespace, table.queueName, table.state, table.availableAt, table.sequence),
45
+ finishedIdx: index("eventually_jobs_finished_idx").on(table.namespace, table.queueName, table.state, table.finishedAt, table.sequence),
46
+ }));
47
+ export class SQLiteAdapter {
48
+ client;
49
+ db;
50
+ namespace;
51
+ schemaPromise = null;
52
+ closed = false;
53
+ constructor(options = {}) {
54
+ this.client = createClient(this.normalizeConnection(options.connection));
55
+ this.db = drizzle(this.client);
56
+ this.namespace = options.namespace ?? "default";
57
+ }
58
+ async enqueue(request) {
59
+ await this.ensureReady();
60
+ return this.withTransaction(async (tx) => {
61
+ const existing = await this.findJob(tx, request.queueName, request.id);
62
+ if (existing) {
63
+ return this.toRecord(existing);
64
+ }
65
+ const [sequence] = await this.allocateSequences(tx, 1);
66
+ if (sequence === undefined) {
67
+ throw new Error("Failed to allocate job sequence.");
68
+ }
69
+ const row = this.createJobRow(request, sequence);
70
+ await tx.insert(jobs).values(row);
71
+ return this.toRecord(row);
72
+ });
73
+ }
74
+ async enqueueMany(requests) {
75
+ await this.ensureReady();
76
+ if (requests.length === 0) {
77
+ return [];
78
+ }
79
+ return this.withTransaction(async (tx) => {
80
+ const existing = new Map();
81
+ const requestedByQueue = new Map();
82
+ for (const request of requests) {
83
+ let ids = requestedByQueue.get(request.queueName);
84
+ if (!ids) {
85
+ ids = new Set();
86
+ requestedByQueue.set(request.queueName, ids);
87
+ }
88
+ ids.add(request.id);
89
+ }
90
+ for (const [queueName, ids] of requestedByQueue) {
91
+ const rows = await tx
92
+ .select()
93
+ .from(jobs)
94
+ .where(and(eq(jobs.namespace, this.namespace), eq(jobs.queueName, queueName), inArray(jobs.jobId, [...ids])));
95
+ for (const row of rows) {
96
+ existing.set(this.jobKey(row.queueName, row.jobId), row);
97
+ }
98
+ }
99
+ const uniqueMissing = new Map();
100
+ for (const request of requests) {
101
+ const key = this.jobKey(request.queueName, request.id);
102
+ if (!existing.has(key) && !uniqueMissing.has(key)) {
103
+ uniqueMissing.set(key, request);
104
+ }
105
+ }
106
+ const missing = [...uniqueMissing.values()];
107
+ const sequences = await this.allocateSequences(tx, missing.length);
108
+ const rowsToInsert = missing.map((request, index) => this.createJobRow(request, sequences[index]));
109
+ if (rowsToInsert.length > 0) {
110
+ await tx.insert(jobs).values(rowsToInsert);
111
+ }
112
+ const inserted = new Map(rowsToInsert.map((row) => [this.jobKey(row.queueName, row.jobId), row]));
113
+ return requests.map((request) => {
114
+ const key = this.jobKey(request.queueName, request.id);
115
+ const row = existing.get(key) ?? inserted.get(key);
116
+ if (!row) {
117
+ throw new Error(`Failed to enqueue job ${key}.`);
118
+ }
119
+ return this.toRecord(row);
120
+ });
121
+ });
122
+ }
123
+ async getJob(queueName, jobId) {
124
+ await this.ensureReady();
125
+ const row = await this.findJob(this.db, queueName, jobId);
126
+ return row ? this.toRecord(row) : null;
127
+ }
128
+ async claimNext(queueName, options) {
129
+ await this.ensureReady();
130
+ return this.withTransaction(async (tx) => {
131
+ const candidates = await tx
132
+ .select()
133
+ .from(jobs)
134
+ .where(and(eq(jobs.namespace, this.namespace), eq(jobs.queueName, queueName), or(eq(jobs.state, "waiting"), and(eq(jobs.state, "delayed"), lte(jobs.availableAt, options.now)))))
135
+ .orderBy(desc(jobs.priority), asc(jobs.sequence))
136
+ .limit(options.limit);
137
+ const claimed = [];
138
+ for (const row of candidates) {
139
+ const updated = await tx
140
+ .update(jobs)
141
+ .set({
142
+ state: "active",
143
+ attempt: row.attempt + 1,
144
+ processedAt: row.processedAt ?? options.now,
145
+ lockToken: this.createLockToken(row, options.now),
146
+ availableAt: options.now + options.lockTtlMs,
147
+ finishedAt: null,
148
+ })
149
+ .where(and(this.matchJob(row.queueName, row.jobId), eq(jobs.state, row.state), row.state === "delayed" ? lte(jobs.availableAt, options.now) : undefined))
150
+ .returning();
151
+ const next = updated[0];
152
+ if (next) {
153
+ claimed.push(this.toRecord(next));
154
+ }
155
+ }
156
+ return claimed;
157
+ });
158
+ }
159
+ async complete(queueName, jobId, lockToken, result, finishedAt, keep) {
160
+ await this.ensureReady();
161
+ await this.withTransaction(async (tx) => {
162
+ const row = await this.requireJob(tx, queueName, jobId);
163
+ this.assertLock(row, lockToken);
164
+ const updated = {
165
+ ...row,
166
+ state: "completed",
167
+ finishedAt,
168
+ result: this.encode(result),
169
+ lockToken: null,
170
+ availableAt: finishedAt,
171
+ };
172
+ if (keep.mode === "none") {
173
+ await tx.delete(jobs).where(this.matchJob(updated.queueName, updated.jobId));
174
+ }
175
+ else {
176
+ await this.writeJob(tx, updated);
177
+ await this.trimTerminal(tx, queueName, "completed", keep);
178
+ }
179
+ await this.resolveParent(tx, updated.parentKey, finishedAt, null);
180
+ });
181
+ }
182
+ async fail(queueName, jobId, lockToken, failure, finishedAt, retryAt, keep) {
183
+ await this.ensureReady();
184
+ await this.withTransaction(async (tx) => {
185
+ const row = await this.requireJob(tx, queueName, jobId);
186
+ this.assertLock(row, lockToken);
187
+ if (retryAt !== null) {
188
+ const retried = {
189
+ ...row,
190
+ state: retryAt <= finishedAt ? "waiting" : "delayed",
191
+ finishedAt: null,
192
+ failedReason: failure.message,
193
+ stack: failure.stack,
194
+ result: null,
195
+ lockToken: null,
196
+ availableAt: retryAt,
197
+ };
198
+ await this.writeJob(tx, retried);
199
+ return;
200
+ }
201
+ const failed = {
202
+ ...row,
203
+ state: "failed",
204
+ finishedAt,
205
+ failedReason: failure.message,
206
+ stack: failure.stack,
207
+ result: null,
208
+ lockToken: null,
209
+ availableAt: finishedAt,
210
+ };
211
+ if (keep.mode === "none") {
212
+ await tx.delete(jobs).where(this.matchJob(failed.queueName, failed.jobId));
213
+ }
214
+ else {
215
+ await this.writeJob(tx, failed);
216
+ await this.trimTerminal(tx, queueName, "failed", keep);
217
+ }
218
+ await this.resolveParent(tx, failed.parentKey, finishedAt, failure.message);
219
+ });
220
+ }
221
+ async updateProgress(queueName, jobId, lockToken, progress, updatedAt) {
222
+ void updatedAt;
223
+ await this.ensureReady();
224
+ await this.withTransaction(async (tx) => {
225
+ const row = await this.requireJob(tx, queueName, jobId);
226
+ this.assertLock(row, lockToken);
227
+ await tx
228
+ .update(jobs)
229
+ .set({
230
+ progress: this.encode(progress),
231
+ })
232
+ .where(this.matchJob(queueName, jobId));
233
+ });
234
+ }
235
+ async recoverStalled(queueName, now, lockTtlMs, limit) {
236
+ void lockTtlMs;
237
+ await this.ensureReady();
238
+ return this.withTransaction(async (tx) => {
239
+ const stalled = await tx
240
+ .select()
241
+ .from(jobs)
242
+ .where(and(eq(jobs.namespace, this.namespace), eq(jobs.queueName, queueName), eq(jobs.state, "active"), lte(jobs.availableAt, now)))
243
+ .orderBy(asc(jobs.availableAt), asc(jobs.sequence))
244
+ .limit(limit);
245
+ const recovered = [];
246
+ for (const row of stalled) {
247
+ if (row.attempt < row.attempts) {
248
+ const retryAt = this.computeRetryAt(row, now);
249
+ const retried = {
250
+ ...row,
251
+ state: retryAt <= now ? "waiting" : "delayed",
252
+ lockToken: null,
253
+ finishedAt: null,
254
+ availableAt: retryAt,
255
+ };
256
+ await this.writeJob(tx, retried);
257
+ recovered.push(this.toRecord(retried));
258
+ continue;
259
+ }
260
+ const message = row.failedReason ?? "stalled job failed";
261
+ const failed = {
262
+ ...row,
263
+ state: "failed",
264
+ finishedAt: now,
265
+ failedReason: message,
266
+ lockToken: null,
267
+ availableAt: now,
268
+ };
269
+ const keep = this.decodeKeep(row.removeOnFail);
270
+ if (keep.mode === "none") {
271
+ await tx.delete(jobs).where(this.matchJob(row.queueName, row.jobId));
272
+ }
273
+ else {
274
+ await this.writeJob(tx, failed);
275
+ await this.trimTerminal(tx, queueName, "failed", keep);
276
+ }
277
+ await this.resolveParent(tx, row.parentKey, now, message);
278
+ recovered.push(this.toRecord(failed));
279
+ }
280
+ return recovered;
281
+ });
282
+ }
283
+ async migrate() {
284
+ await this.ensureReady();
285
+ }
286
+ async close() {
287
+ if (this.closed) {
288
+ return;
289
+ }
290
+ this.closed = true;
291
+ await this.client.close?.();
292
+ }
293
+ async ensureReady() {
294
+ if (this.closed) {
295
+ throw new Error("SQLiteAdapter is closed.");
296
+ }
297
+ if (!this.schemaPromise) {
298
+ this.schemaPromise = this.createSchema();
299
+ }
300
+ await this.schemaPromise;
301
+ }
302
+ async createSchema() {
303
+ await this.client.execute(`
304
+ CREATE TABLE IF NOT EXISTS eventually_counters (
305
+ namespace TEXT NOT NULL,
306
+ key TEXT NOT NULL,
307
+ value INTEGER NOT NULL,
308
+ PRIMARY KEY (namespace, key)
309
+ )
310
+ `);
311
+ await this.client.execute(`
312
+ CREATE TABLE IF NOT EXISTS eventually_jobs (
313
+ namespace TEXT NOT NULL,
314
+ queue_name TEXT NOT NULL,
315
+ job_id TEXT NOT NULL,
316
+ name TEXT NOT NULL,
317
+ data TEXT NOT NULL,
318
+ state TEXT NOT NULL,
319
+ attempt INTEGER NOT NULL,
320
+ attempts INTEGER NOT NULL,
321
+ priority INTEGER NOT NULL,
322
+ delay INTEGER NOT NULL,
323
+ created_at INTEGER NOT NULL,
324
+ processed_at INTEGER,
325
+ finished_at INTEGER,
326
+ result TEXT,
327
+ failed_reason TEXT,
328
+ stack TEXT,
329
+ progress TEXT,
330
+ lock_token TEXT,
331
+ available_at INTEGER NOT NULL,
332
+ backoff TEXT,
333
+ remove_on_complete TEXT NOT NULL,
334
+ remove_on_fail TEXT NOT NULL,
335
+ parent_key TEXT,
336
+ pending_children INTEGER NOT NULL,
337
+ child_failure TEXT,
338
+ sequence INTEGER NOT NULL,
339
+ PRIMARY KEY (namespace, queue_name, job_id)
340
+ )
341
+ `);
342
+ await this.client.execute(`
343
+ CREATE INDEX IF NOT EXISTS eventually_jobs_waiting_idx
344
+ ON eventually_jobs (namespace, queue_name, state, priority DESC, sequence ASC)
345
+ `);
346
+ await this.client.execute(`
347
+ CREATE INDEX IF NOT EXISTS eventually_jobs_available_idx
348
+ ON eventually_jobs (namespace, queue_name, state, available_at ASC, sequence ASC)
349
+ `);
350
+ await this.client.execute(`
351
+ CREATE INDEX IF NOT EXISTS eventually_jobs_finished_idx
352
+ ON eventually_jobs (namespace, queue_name, state, finished_at ASC, sequence ASC)
353
+ `);
354
+ }
355
+ async allocateSequences(tx, countNeeded) {
356
+ if (countNeeded === 0) {
357
+ return [];
358
+ }
359
+ await tx
360
+ .insert(counters)
361
+ .values({
362
+ namespace: this.namespace,
363
+ key: SEQUENCE_KEY,
364
+ value: 0,
365
+ })
366
+ .onConflictDoNothing();
367
+ await tx
368
+ .update(counters)
369
+ .set({
370
+ value: sql `${counters.value} + ${countNeeded}`,
371
+ })
372
+ .where(and(eq(counters.namespace, this.namespace), eq(counters.key, SEQUENCE_KEY)));
373
+ const [row] = await tx
374
+ .select()
375
+ .from(counters)
376
+ .where(and(eq(counters.namespace, this.namespace), eq(counters.key, SEQUENCE_KEY)))
377
+ .limit(1);
378
+ const end = row?.value;
379
+ if (end === undefined) {
380
+ throw new Error("Failed to read allocated sequence.");
381
+ }
382
+ return Array.from({ length: countNeeded }, (_, index) => end - countNeeded + index + 1);
383
+ }
384
+ createJobRow(request, sequence) {
385
+ const state = (request.pendingChildren ?? 0) > 0
386
+ ? "blocked"
387
+ : request.delay > 0
388
+ ? "delayed"
389
+ : "waiting";
390
+ return {
391
+ namespace: this.namespace,
392
+ queueName: request.queueName,
393
+ jobId: request.id,
394
+ name: request.name,
395
+ data: this.encode(request.data),
396
+ state,
397
+ attempt: 0,
398
+ attempts: request.attempts,
399
+ priority: request.priority,
400
+ delay: request.delay,
401
+ createdAt: request.createdAt,
402
+ processedAt: null,
403
+ finishedAt: null,
404
+ result: null,
405
+ failedReason: null,
406
+ stack: null,
407
+ progress: null,
408
+ lockToken: null,
409
+ availableAt: request.createdAt + request.delay,
410
+ backoff: request.backoff ? this.encode(request.backoff) : null,
411
+ removeOnComplete: this.encode(request.removeOnComplete),
412
+ removeOnFail: this.encode(request.removeOnFail),
413
+ parentKey: request.parentKey ?? null,
414
+ pendingChildren: request.pendingChildren ?? 0,
415
+ childFailure: null,
416
+ sequence,
417
+ };
418
+ }
419
+ toRecord(row) {
420
+ return {
421
+ id: row.jobId,
422
+ queueName: row.queueName,
423
+ name: row.name,
424
+ data: this.decode(row.data),
425
+ state: row.state,
426
+ attempt: row.attempt,
427
+ attempts: row.attempts,
428
+ priority: row.priority,
429
+ delay: row.delay,
430
+ createdAt: row.createdAt,
431
+ processedAt: row.processedAt,
432
+ finishedAt: row.finishedAt,
433
+ result: this.decode(row.result),
434
+ failedReason: row.failedReason,
435
+ stack: row.stack,
436
+ progress: this.decode(row.progress),
437
+ lockToken: row.lockToken,
438
+ backoff: this.decodeBackoff(row.backoff),
439
+ removeOnComplete: this.decodeKeep(row.removeOnComplete),
440
+ removeOnFail: this.decodeKeep(row.removeOnFail),
441
+ parentKey: row.parentKey,
442
+ pendingChildren: row.pendingChildren,
443
+ childFailure: row.childFailure,
444
+ };
445
+ }
446
+ async findJob(tx, queueName, jobId) {
447
+ const [row] = await tx
448
+ .select()
449
+ .from(jobs)
450
+ .where(this.matchJob(queueName, jobId))
451
+ .limit(1);
452
+ return row ?? null;
453
+ }
454
+ async requireJob(tx, queueName, jobId) {
455
+ const row = await this.findJob(tx, queueName, jobId);
456
+ if (!row) {
457
+ throw new Error(`Job ${jobId} was not found in queue ${queueName}.`);
458
+ }
459
+ return row;
460
+ }
461
+ assertLock(row, lockToken) {
462
+ if (row.lockToken !== lockToken) {
463
+ throw new Error("lock-mismatch");
464
+ }
465
+ }
466
+ createLockToken(row, now) {
467
+ return `${row.queueName}:${row.jobId}:${row.attempt + 1}:${now}`;
468
+ }
469
+ async writeJob(tx, row) {
470
+ await tx
471
+ .update(jobs)
472
+ .set({
473
+ state: row.state,
474
+ attempt: row.attempt,
475
+ processedAt: row.processedAt,
476
+ finishedAt: row.finishedAt,
477
+ result: row.result,
478
+ failedReason: row.failedReason,
479
+ stack: row.stack,
480
+ progress: row.progress,
481
+ lockToken: row.lockToken,
482
+ availableAt: row.availableAt,
483
+ backoff: row.backoff,
484
+ removeOnComplete: row.removeOnComplete,
485
+ removeOnFail: row.removeOnFail,
486
+ parentKey: row.parentKey,
487
+ pendingChildren: row.pendingChildren,
488
+ childFailure: row.childFailure,
489
+ })
490
+ .where(this.matchJob(row.queueName, row.jobId));
491
+ }
492
+ async trimTerminal(tx, queueName, state, keep) {
493
+ if (keep.mode !== "count") {
494
+ return;
495
+ }
496
+ const [counter] = await tx
497
+ .select({ value: count() })
498
+ .from(jobs)
499
+ .where(and(eq(jobs.namespace, this.namespace), eq(jobs.queueName, queueName), eq(jobs.state, state)));
500
+ const current = Number(counter?.value ?? 0);
501
+ const limit = keep.count ?? 0;
502
+ const overflow = current - limit;
503
+ if (overflow <= 0) {
504
+ return;
505
+ }
506
+ const doomed = await tx
507
+ .select({ queueName: jobs.queueName, jobId: jobs.jobId })
508
+ .from(jobs)
509
+ .where(and(eq(jobs.namespace, this.namespace), eq(jobs.queueName, queueName), eq(jobs.state, state)))
510
+ .orderBy(asc(jobs.finishedAt), asc(jobs.sequence))
511
+ .limit(overflow);
512
+ for (const row of doomed) {
513
+ await tx.delete(jobs).where(this.matchJob(row.queueName, row.jobId));
514
+ }
515
+ }
516
+ async resolveParent(tx, parentKey, finishedAt, failureMessage) {
517
+ let currentParentKey = parentKey;
518
+ let propagatedFailure = failureMessage;
519
+ while (currentParentKey) {
520
+ const parsed = this.parseParentKey(currentParentKey);
521
+ if (!parsed) {
522
+ return;
523
+ }
524
+ const parent = await this.findJob(tx, parsed.queueName, parsed.jobId);
525
+ if (!parent) {
526
+ return;
527
+ }
528
+ parent.pendingChildren = Math.max(0, parent.pendingChildren - 1);
529
+ if (propagatedFailure) {
530
+ parent.childFailure = propagatedFailure;
531
+ }
532
+ if (parent.pendingChildren > 0) {
533
+ await this.writeJob(tx, parent);
534
+ return;
535
+ }
536
+ if (parent.childFailure) {
537
+ parent.state = "failed";
538
+ parent.finishedAt = finishedAt;
539
+ parent.failedReason = parent.childFailure;
540
+ parent.lockToken = null;
541
+ parent.availableAt = finishedAt;
542
+ const keep = this.decodeKeep(parent.removeOnFail);
543
+ if (keep.mode === "none") {
544
+ await tx.delete(jobs).where(this.matchJob(parent.queueName, parent.jobId));
545
+ }
546
+ else {
547
+ await this.writeJob(tx, parent);
548
+ await this.trimTerminal(tx, parent.queueName, "failed", keep);
549
+ }
550
+ propagatedFailure = parent.childFailure;
551
+ currentParentKey = parent.parentKey;
552
+ continue;
553
+ }
554
+ if (parent.delay > 0 && parent.createdAt + parent.delay > finishedAt) {
555
+ parent.state = "delayed";
556
+ parent.availableAt = parent.createdAt + parent.delay;
557
+ }
558
+ else {
559
+ parent.state = "waiting";
560
+ parent.availableAt = finishedAt;
561
+ }
562
+ await this.writeJob(tx, parent);
563
+ return;
564
+ }
565
+ }
566
+ computeRetryAt(row, now) {
567
+ const backoff = this.decodeBackoff(row.backoff);
568
+ if (!backoff) {
569
+ return now;
570
+ }
571
+ if (backoff.type === "fixed") {
572
+ return now + backoff.delayMs;
573
+ }
574
+ const delay = backoff.delayMs * 2 ** Math.max(0, row.attempt - 1);
575
+ return now + Math.min(delay, backoff.maxDelayMs ?? delay);
576
+ }
577
+ parseParentKey(parentKey) {
578
+ const separator = parentKey.indexOf(":");
579
+ if (separator < 1 || separator === parentKey.length - 1) {
580
+ return null;
581
+ }
582
+ return {
583
+ queueName: parentKey.slice(0, separator),
584
+ jobId: parentKey.slice(separator + 1),
585
+ };
586
+ }
587
+ matchJob(queueName, jobId) {
588
+ return and(eq(jobs.namespace, this.namespace), eq(jobs.queueName, queueName), eq(jobs.jobId, jobId));
589
+ }
590
+ jobKey(queueName, jobId) {
591
+ return `${queueName}:${jobId}`;
592
+ }
593
+ encode(value) {
594
+ return JSON.stringify(value ?? null);
595
+ }
596
+ decode(value) {
597
+ return value === null ? null : JSON.parse(value);
598
+ }
599
+ decodeBackoff(value) {
600
+ return this.decode(value);
601
+ }
602
+ decodeKeep(value) {
603
+ return JSON.parse(value);
604
+ }
605
+ async withTransaction(callback) {
606
+ let attempt = 0;
607
+ while (true) {
608
+ try {
609
+ return await this.db.transaction((tx) => callback(tx));
610
+ }
611
+ catch (error) {
612
+ if (!this.isRetryableSqliteError(error) || attempt >= TRANSACTION_MAX_RETRIES) {
613
+ throw error;
614
+ }
615
+ attempt += 1;
616
+ await this.sleep(Math.min(10, attempt));
617
+ }
618
+ }
619
+ }
620
+ normalizeConnection(connection) {
621
+ if (typeof connection === "string") {
622
+ return { url: this.normalizeUrl(connection) };
623
+ }
624
+ if (connection) {
625
+ return {
626
+ ...connection,
627
+ url: this.normalizeUrl(connection.url),
628
+ };
629
+ }
630
+ return { url: this.createInMemoryUrl() };
631
+ }
632
+ normalizeUrl(url) {
633
+ return url === ":memory:" ? this.createInMemoryUrl() : url;
634
+ }
635
+ createInMemoryUrl() {
636
+ return "file::memory:?cache=shared";
637
+ }
638
+ isRetryableSqliteError(error) {
639
+ if (!(error instanceof Error)) {
640
+ return false;
641
+ }
642
+ const code = "code" in error ? String(error.code) : "";
643
+ return code.startsWith("SQLITE_BUSY") || code.startsWith("SQLITE_LOCKED");
644
+ }
645
+ sleep(ms) {
646
+ return new Promise((resolve) => setTimeout(resolve, ms));
647
+ }
648
+ }
package/package.json ADDED
@@ -0,0 +1,39 @@
1
+ {
2
+ "name": "@ventually/sqlite",
3
+ "version": "0.0.1",
4
+ "type": "module",
5
+ "bin": {
6
+ "@ventually/sqlite": "./dist/cli.js"
7
+ },
8
+ "exports": {
9
+ ".": {
10
+ "types": "./dist/index.d.ts",
11
+ "default": "./dist/index.js"
12
+ }
13
+ },
14
+ "main": "./dist/index.js",
15
+ "types": "./dist/index.d.ts",
16
+ "files": [
17
+ "dist"
18
+ ],
19
+ "publishConfig": {
20
+ "access": "public"
21
+ },
22
+ "scripts": {
23
+ "build": "rm -rf dist && tsc -p tsconfig.build.json",
24
+ "check-types": "tsc --noEmit -p tsconfig.json",
25
+ "test": "vitest run --project unit",
26
+ "test:stress": "bun run ./test/sqlite.stress.ts"
27
+ },
28
+ "dependencies": {
29
+ "@libsql/client": "^0.15.15",
30
+ "drizzle-orm": "^0.44.5",
31
+ "@ventually/core": "*"
32
+ },
33
+ "devDependencies": {
34
+ "@repo/typescript-config": "*",
35
+ "@types/node": "^25.5.2",
36
+ "typescript": "5.9.2",
37
+ "vitest": "^3.2.4"
38
+ }
39
+ }