@workglow/indexeddb 0.2.31 → 0.2.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -0
- package/dist/job-queue/IndexedDbQueueStorage.d.ts +16 -11
- package/dist/job-queue/IndexedDbQueueStorage.d.ts.map +1 -1
- package/dist/job-queue/IndexedDbRateLimiterStorage.d.ts +15 -4
- package/dist/job-queue/IndexedDbRateLimiterStorage.d.ts.map +1 -1
- package/dist/job-queue/browser.js +399 -351
- package/dist/job-queue/browser.js.map +9 -6
- package/dist/job-queue/common.d.ts +3 -0
- package/dist/job-queue/common.d.ts.map +1 -1
- package/dist/job-queue/node.js +399 -351
- package/dist/job-queue/node.js.map +9 -6
- package/dist/migrations/IndexedDbMigrationRunner.d.ts +93 -0
- package/dist/migrations/IndexedDbMigrationRunner.d.ts.map +1 -0
- package/dist/migrations/indexedDbQueueMigrations.d.ts +24 -0
- package/dist/migrations/indexedDbQueueMigrations.d.ts.map +1 -0
- package/dist/migrations/indexedDbRateLimiterMigrations.d.ts +37 -0
- package/dist/migrations/indexedDbRateLimiterMigrations.d.ts.map +1 -0
- package/dist/storage/IndexedDbTable.d.ts.map +1 -1
- package/dist/storage/IndexedDbTabularMigrationApplier.d.ts +84 -0
- package/dist/storage/IndexedDbTabularMigrationApplier.d.ts.map +1 -0
- package/dist/storage/IndexedDbTabularStorage.d.ts +21 -2
- package/dist/storage/IndexedDbTabularStorage.d.ts.map +1 -1
- package/dist/storage/browser.js +472 -30
- package/dist/storage/browser.js.map +8 -5
- package/dist/storage/common.d.ts +3 -0
- package/dist/storage/common.d.ts.map +1 -1
- package/dist/storage/node.js +472 -30
- package/dist/storage/node.js.map +8 -5
- package/dist/storage/openIdb.d.ts +19 -0
- package/dist/storage/openIdb.d.ts.map +1 -0
- package/package.json +7 -7
package/dist/job-queue/node.js
CHANGED
|
@@ -1,319 +1,330 @@
|
|
|
1
1
|
// src/job-queue/IndexedDbQueueStorage.ts
|
|
2
|
-
import { createServiceToken, deepEqual
|
|
2
|
+
import { createServiceToken, deepEqual, makeFingerprint, uuid4 } from "@workglow/util";
|
|
3
3
|
import { HybridSubscriptionManager } from "@workglow/storage";
|
|
4
4
|
|
|
5
|
-
// src/storage/
|
|
6
|
-
|
|
7
|
-
var METADATA_STORE_NAME = "__schema_metadata__";
|
|
8
|
-
async function saveSchemaMetadata(db, tableName, snapshot) {
|
|
5
|
+
// src/storage/openIdb.ts
|
|
6
|
+
function openIdb(dbName, options = {}) {
|
|
9
7
|
return new Promise((resolve, reject) => {
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
const
|
|
13
|
-
|
|
14
|
-
request.onsuccess = () => resolve();
|
|
15
|
-
request.onerror = () => reject(request.error);
|
|
16
|
-
transaction.onerror = () => reject(transaction.error);
|
|
17
|
-
} catch (err) {
|
|
18
|
-
resolve();
|
|
19
|
-
}
|
|
20
|
-
});
|
|
21
|
-
}
|
|
22
|
-
async function openIndexedDbTable(tableName, version, upgradeNeededCallback) {
|
|
23
|
-
return new Promise((resolve, reject) => {
|
|
24
|
-
const openRequest = indexedDB.open(tableName, version);
|
|
25
|
-
openRequest.onsuccess = (event) => {
|
|
26
|
-
const db = event.target.result;
|
|
27
|
-
db.onversionchange = () => {
|
|
28
|
-
db.close();
|
|
29
|
-
};
|
|
8
|
+
const req = indexedDB.open(dbName, options.version);
|
|
9
|
+
req.onsuccess = () => {
|
|
10
|
+
const db = req.result;
|
|
11
|
+
db.onversionchange = () => db.close();
|
|
30
12
|
resolve(db);
|
|
31
13
|
};
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
const error = openRequest.error;
|
|
39
|
-
if (error && error.name === "VersionError") {
|
|
40
|
-
reject(new Error(`Database ${tableName} exists at a higher version. Cannot open at version ${version || "current"}.`));
|
|
41
|
-
} else {
|
|
42
|
-
reject(error);
|
|
14
|
+
req.onupgradeneeded = (ev) => options.onUpgradeNeeded?.(ev);
|
|
15
|
+
req.onerror = () => {
|
|
16
|
+
const err = req.error;
|
|
17
|
+
if (err && err.name === "VersionError") {
|
|
18
|
+
reject(new Error(`IndexedDB ${dbName} exists at a higher version than ${options.version ?? "current"}`));
|
|
19
|
+
return;
|
|
43
20
|
}
|
|
21
|
+
reject(err);
|
|
44
22
|
};
|
|
45
|
-
|
|
46
|
-
reject(new Error(`Database ${tableName} is blocked. Close all other tabs using this database.`));
|
|
47
|
-
};
|
|
23
|
+
req.onblocked = () => reject(new Error(`IndexedDB ${dbName} is blocked — close other tabs using this database.`));
|
|
48
24
|
});
|
|
49
25
|
}
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
function compareSchemas(store, expectedPrimaryKey, expectedIndexes) {
|
|
61
|
-
const diff = {
|
|
62
|
-
indexesToAdd: [],
|
|
63
|
-
indexesToRemove: [],
|
|
64
|
-
indexesToModify: [],
|
|
65
|
-
primaryKeyChanged: false,
|
|
66
|
-
needsObjectStoreRecreation: false
|
|
67
|
-
};
|
|
68
|
-
const actualKeyPath = store.keyPath;
|
|
69
|
-
const normalizedExpected = Array.isArray(expectedPrimaryKey) ? expectedPrimaryKey : expectedPrimaryKey;
|
|
70
|
-
const normalizedActual = Array.isArray(actualKeyPath) ? actualKeyPath : actualKeyPath;
|
|
71
|
-
if (!deepEqual(normalizedExpected, normalizedActual)) {
|
|
72
|
-
diff.primaryKeyChanged = true;
|
|
73
|
-
diff.needsObjectStoreRecreation = true;
|
|
74
|
-
return diff;
|
|
75
|
-
}
|
|
76
|
-
const existingIndexes = new Map;
|
|
77
|
-
for (let i = 0;i < store.indexNames.length; i++) {
|
|
78
|
-
const indexName = store.indexNames[i];
|
|
79
|
-
existingIndexes.set(indexName, store.index(indexName));
|
|
80
|
-
}
|
|
81
|
-
for (const expectedIdx of expectedIndexes) {
|
|
82
|
-
const existingIdx = existingIndexes.get(expectedIdx.name);
|
|
83
|
-
if (!existingIdx) {
|
|
84
|
-
diff.indexesToAdd.push(expectedIdx);
|
|
85
|
-
} else {
|
|
86
|
-
const expectedKeyPath = Array.isArray(expectedIdx.keyPath) ? expectedIdx.keyPath : [expectedIdx.keyPath];
|
|
87
|
-
const actualKeyPath2 = Array.isArray(existingIdx.keyPath) ? existingIdx.keyPath : [existingIdx.keyPath];
|
|
88
|
-
const keyPathChanged = !deepEqual(expectedKeyPath, actualKeyPath2);
|
|
89
|
-
const uniqueChanged = existingIdx.unique !== (expectedIdx.options?.unique ?? false);
|
|
90
|
-
const multiEntryChanged = existingIdx.multiEntry !== (expectedIdx.options?.multiEntry ?? false);
|
|
91
|
-
if (keyPathChanged || uniqueChanged || multiEntryChanged) {
|
|
92
|
-
diff.indexesToModify.push(expectedIdx);
|
|
93
|
-
}
|
|
94
|
-
existingIndexes.delete(expectedIdx.name);
|
|
95
|
-
}
|
|
26
|
+
|
|
27
|
+
// src/migrations/IndexedDbMigrationRunner.ts
|
|
28
|
+
import {
|
|
29
|
+
MIGRATIONS_TABLE,
|
|
30
|
+
sortMigrations
|
|
31
|
+
} from "@workglow/storage";
|
|
32
|
+
function getIndexedDb() {
|
|
33
|
+
const idb = globalThis.indexedDB;
|
|
34
|
+
if (!idb) {
|
|
35
|
+
throw new Error("indexedDB is not available in this environment. Provide one via the IndexedDbMigrationRunner constructor or polyfill globalThis.indexedDB.");
|
|
96
36
|
}
|
|
97
|
-
|
|
98
|
-
return diff;
|
|
99
|
-
}
|
|
100
|
-
async function readAllData(store) {
|
|
101
|
-
return new Promise((resolve, reject) => {
|
|
102
|
-
const request = store.getAll();
|
|
103
|
-
request.onsuccess = () => resolve(request.result || []);
|
|
104
|
-
request.onerror = () => reject(request.error);
|
|
105
|
-
});
|
|
106
|
-
}
|
|
107
|
-
async function performIncrementalMigration(db, tableName, diff, options = {}) {
|
|
108
|
-
const currentVersion = db.version;
|
|
109
|
-
const newVersion = currentVersion + 1;
|
|
110
|
-
db.close();
|
|
111
|
-
options.onMigrationProgress?.(`Migrating ${tableName} from version ${currentVersion} to ${newVersion}...`, 0);
|
|
112
|
-
return openIndexedDbTable(tableName, newVersion, (event) => {
|
|
113
|
-
const transaction = event.target.transaction;
|
|
114
|
-
const store = transaction.objectStore(tableName);
|
|
115
|
-
for (const indexName of diff.indexesToRemove) {
|
|
116
|
-
options.onMigrationProgress?.(`Removing index: ${indexName}`, 0.2);
|
|
117
|
-
store.deleteIndex(indexName);
|
|
118
|
-
}
|
|
119
|
-
for (const indexDef of diff.indexesToModify) {
|
|
120
|
-
options.onMigrationProgress?.(`Updating index: ${indexDef.name}`, 0.4);
|
|
121
|
-
if (store.indexNames.contains(indexDef.name)) {
|
|
122
|
-
store.deleteIndex(indexDef.name);
|
|
123
|
-
}
|
|
124
|
-
store.createIndex(indexDef.name, indexDef.keyPath, indexDef.options);
|
|
125
|
-
}
|
|
126
|
-
for (const indexDef of diff.indexesToAdd) {
|
|
127
|
-
options.onMigrationProgress?.(`Adding index: ${indexDef.name}`, 0.6);
|
|
128
|
-
store.createIndex(indexDef.name, indexDef.keyPath, indexDef.options);
|
|
129
|
-
}
|
|
130
|
-
options.onMigrationProgress?.(`Migration complete`, 1);
|
|
131
|
-
});
|
|
37
|
+
return idb;
|
|
132
38
|
}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
39
|
+
|
|
40
|
+
class MigrationAbortedByOtherTabError extends Error {
|
|
41
|
+
constructor(dbName) {
|
|
42
|
+
super(`IndexedDB ${dbName} migration aborted: another tab requested a higher version`);
|
|
43
|
+
this.name = "MigrationAbortedByOtherTabError";
|
|
136
44
|
}
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
} catch (err) {
|
|
147
|
-
options.onMigrationWarning?.(`Failed to read existing data during migration: ${err}`, err);
|
|
45
|
+
}
|
|
46
|
+
var RUN_LOCKS = new Map;
|
|
47
|
+
|
|
48
|
+
class IndexedDbMigrationRunner {
|
|
49
|
+
dbName;
|
|
50
|
+
idb;
|
|
51
|
+
constructor(dbName, idb = getIndexedDb()) {
|
|
52
|
+
this.dbName = dbName;
|
|
53
|
+
this.idb = idb;
|
|
148
54
|
}
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
55
|
+
async probe() {
|
|
56
|
+
return new Promise((resolve, reject) => {
|
|
57
|
+
let settled = false;
|
|
58
|
+
const finalize = (db, outcome) => {
|
|
59
|
+
if (settled)
|
|
60
|
+
return;
|
|
61
|
+
settled = true;
|
|
62
|
+
if (db) {
|
|
63
|
+
try {
|
|
64
|
+
db.close();
|
|
65
|
+
} catch {}
|
|
159
66
|
}
|
|
160
|
-
if (
|
|
161
|
-
|
|
67
|
+
if (outcome.ok)
|
|
68
|
+
resolve(outcome.value);
|
|
69
|
+
else
|
|
70
|
+
reject(outcome.error);
|
|
71
|
+
};
|
|
72
|
+
const req = this.idb.open(this.dbName);
|
|
73
|
+
req.onupgradeneeded = () => {
|
|
74
|
+
if (settled)
|
|
75
|
+
return;
|
|
76
|
+
const u = req.result;
|
|
77
|
+
if (!u.objectStoreNames.contains(MIGRATIONS_TABLE)) {
|
|
78
|
+
u.createObjectStore(MIGRATIONS_TABLE, { keyPath: ["component", "version"] });
|
|
162
79
|
}
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
80
|
+
};
|
|
81
|
+
req.onsuccess = () => {
|
|
82
|
+
if (settled)
|
|
83
|
+
return;
|
|
84
|
+
const db = req.result;
|
|
85
|
+
const currentVersion = db.version;
|
|
86
|
+
if (!db.objectStoreNames.contains(MIGRATIONS_TABLE)) {
|
|
87
|
+
finalize(db, { ok: true, value: { currentVersion, applied: new Set } });
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
const tx = db.transaction(MIGRATIONS_TABLE, "readonly");
|
|
91
|
+
const store = tx.objectStore(MIGRATIONS_TABLE);
|
|
92
|
+
const getAll = store.getAll();
|
|
93
|
+
getAll.onsuccess = () => {
|
|
94
|
+
if (settled)
|
|
95
|
+
return;
|
|
96
|
+
const rows = getAll.result;
|
|
97
|
+
finalize(db, {
|
|
98
|
+
ok: true,
|
|
99
|
+
value: {
|
|
100
|
+
currentVersion,
|
|
101
|
+
applied: new Set(rows.map((r) => `${r.component}@${r.version}`))
|
|
102
|
+
}
|
|
103
|
+
});
|
|
104
|
+
};
|
|
105
|
+
getAll.onerror = () => {
|
|
106
|
+
if (settled)
|
|
107
|
+
return;
|
|
108
|
+
finalize(db, { ok: false, error: getAll.error });
|
|
109
|
+
};
|
|
110
|
+
};
|
|
111
|
+
req.onerror = () => {
|
|
112
|
+
if (settled)
|
|
113
|
+
return;
|
|
114
|
+
finalize(undefined, { ok: false, error: req.error });
|
|
115
|
+
};
|
|
116
|
+
req.onblocked = () => {
|
|
117
|
+
if (settled)
|
|
118
|
+
return;
|
|
119
|
+
finalize(undefined, {
|
|
120
|
+
ok: false,
|
|
121
|
+
error: new Error(`IndexedDB ${this.dbName} blocked while probing for bookkeeping store`)
|
|
122
|
+
});
|
|
123
|
+
};
|
|
124
|
+
});
|
|
170
125
|
}
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
126
|
+
async ensureBookkeepingTable() {
|
|
127
|
+
await this.probe();
|
|
128
|
+
}
|
|
129
|
+
async appliedVersions(component) {
|
|
130
|
+
const { applied } = await this.probe();
|
|
131
|
+
const versions = new Set;
|
|
132
|
+
for (const key of applied) {
|
|
133
|
+
const at = key.lastIndexOf("@");
|
|
134
|
+
if (at < 0)
|
|
135
|
+
continue;
|
|
136
|
+
const c = key.slice(0, at);
|
|
137
|
+
const v = Number(key.slice(at + 1));
|
|
138
|
+
if (c === component && Number.isFinite(v))
|
|
139
|
+
versions.add(v);
|
|
176
140
|
}
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
141
|
+
return versions;
|
|
142
|
+
}
|
|
143
|
+
async run(migrations, options = {}) {
|
|
144
|
+
const prev = RUN_LOCKS.get(this.dbName) ?? Promise.resolve();
|
|
145
|
+
const next = prev.catch(() => {
|
|
146
|
+
return;
|
|
147
|
+
}).then(() => this.runLocked(migrations, options));
|
|
148
|
+
RUN_LOCKS.set(this.dbName, next);
|
|
149
|
+
try {
|
|
150
|
+
return await next;
|
|
151
|
+
} finally {
|
|
152
|
+
if (RUN_LOCKS.get(this.dbName) === next)
|
|
153
|
+
RUN_LOCKS.delete(this.dbName);
|
|
180
154
|
}
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
155
|
+
}
|
|
156
|
+
async runLocked(migrations, options) {
|
|
157
|
+
const sorted = sortMigrations(migrations);
|
|
158
|
+
if (sorted.length === 0)
|
|
159
|
+
return [];
|
|
160
|
+
const { currentVersion, applied: alreadyApplied } = await this.probe();
|
|
161
|
+
const pending = sorted.filter((m) => !alreadyApplied.has(`${m.component}@${m.version}`));
|
|
162
|
+
if (pending.length === 0)
|
|
163
|
+
return [];
|
|
164
|
+
const targetVersion = currentVersion + 1;
|
|
165
|
+
const applied = [];
|
|
166
|
+
const onProgress = options.onProgress;
|
|
167
|
+
const buffered = [];
|
|
168
|
+
const emitLater = onProgress ? (ev) => buffered.push(ev) : undefined;
|
|
169
|
+
await new Promise((resolve, reject) => {
|
|
170
|
+
let settled = false;
|
|
171
|
+
let upgradeDb;
|
|
172
|
+
const finalize = (outcome) => {
|
|
173
|
+
if (settled)
|
|
174
|
+
return;
|
|
175
|
+
settled = true;
|
|
176
|
+
if (upgradeDb) {
|
|
177
|
+
try {
|
|
178
|
+
upgradeDb.close();
|
|
179
|
+
} catch {}
|
|
180
|
+
}
|
|
181
|
+
if (outcome.ok)
|
|
182
|
+
resolve();
|
|
183
|
+
else
|
|
184
|
+
reject(outcome.error);
|
|
185
|
+
};
|
|
186
|
+
const upreq = this.idb.open(this.dbName, targetVersion);
|
|
187
|
+
upreq.onupgradeneeded = (ev) => {
|
|
188
|
+
if (settled)
|
|
189
|
+
return;
|
|
184
190
|
try {
|
|
185
|
-
|
|
191
|
+
const db = upreq.result;
|
|
192
|
+
upgradeDb = db;
|
|
193
|
+
const tx = upreq.transaction;
|
|
194
|
+
const oldVersion = ev.oldVersion;
|
|
195
|
+
const newVersion = ev.newVersion ?? targetVersion;
|
|
196
|
+
db.onversionchange = () => {
|
|
197
|
+
try {
|
|
198
|
+
tx.abort();
|
|
199
|
+
} catch {}
|
|
200
|
+
finalize({ ok: false, error: new MigrationAbortedByOtherTabError(this.dbName) });
|
|
201
|
+
};
|
|
202
|
+
if (!db.objectStoreNames.contains(MIGRATIONS_TABLE)) {
|
|
203
|
+
db.createObjectStore(MIGRATIONS_TABLE, { keyPath: ["component", "version"] });
|
|
204
|
+
}
|
|
205
|
+
const meta = tx.objectStore(MIGRATIONS_TABLE);
|
|
206
|
+
for (const m of pending) {
|
|
207
|
+
emitLater?.({
|
|
208
|
+
component: m.component,
|
|
209
|
+
version: m.version,
|
|
210
|
+
phase: "starting",
|
|
211
|
+
description: m.description
|
|
212
|
+
});
|
|
213
|
+
const ctx = { db, tx, oldVersion, newVersion };
|
|
214
|
+
const result = m.up(ctx, (fraction) => {
|
|
215
|
+
emitLater?.({
|
|
216
|
+
component: m.component,
|
|
217
|
+
version: m.version,
|
|
218
|
+
phase: "running",
|
|
219
|
+
description: m.description,
|
|
220
|
+
fraction
|
|
221
|
+
});
|
|
222
|
+
});
|
|
223
|
+
if (result instanceof Promise) {
|
|
224
|
+
throw new Error(`IndexedDB migration "${m.component}@${m.version}" returned a Promise; ` + `IDB upgrade transactions cannot span async work.`);
|
|
225
|
+
}
|
|
226
|
+
meta.add({
|
|
227
|
+
component: m.component,
|
|
228
|
+
version: m.version,
|
|
229
|
+
description: m.description ?? null,
|
|
230
|
+
applied_at: new Date().toISOString()
|
|
231
|
+
});
|
|
232
|
+
applied.push(m);
|
|
233
|
+
emitLater?.({
|
|
234
|
+
component: m.component,
|
|
235
|
+
version: m.version,
|
|
236
|
+
phase: "completed",
|
|
237
|
+
description: m.description,
|
|
238
|
+
fraction: 1
|
|
239
|
+
});
|
|
240
|
+
}
|
|
186
241
|
} catch (err) {
|
|
187
|
-
|
|
242
|
+
try {
|
|
243
|
+
upreq.transaction?.abort();
|
|
244
|
+
} catch {}
|
|
245
|
+
const lastStart = [...buffered].reverse().find((e) => e.phase === "starting");
|
|
246
|
+
if (lastStart) {
|
|
247
|
+
emitLater?.({
|
|
248
|
+
component: lastStart.component,
|
|
249
|
+
version: lastStart.version,
|
|
250
|
+
phase: "failed",
|
|
251
|
+
description: lastStart.description,
|
|
252
|
+
error: err
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
finalize({ ok: false, error: err });
|
|
188
256
|
}
|
|
257
|
+
};
|
|
258
|
+
upreq.onsuccess = () => {
|
|
259
|
+
if (settled)
|
|
260
|
+
return;
|
|
261
|
+
upgradeDb = upreq.result;
|
|
262
|
+
finalize({ ok: true });
|
|
263
|
+
};
|
|
264
|
+
upreq.onerror = () => {
|
|
265
|
+
if (settled)
|
|
266
|
+
return;
|
|
267
|
+
finalize({ ok: false, error: upreq.error });
|
|
268
|
+
};
|
|
269
|
+
upreq.onblocked = () => {
|
|
270
|
+
if (settled)
|
|
271
|
+
return;
|
|
272
|
+
finalize({
|
|
273
|
+
ok: false,
|
|
274
|
+
error: new Error(`IndexedDB ${this.dbName} upgrade blocked — close other tabs.`)
|
|
275
|
+
});
|
|
276
|
+
};
|
|
277
|
+
}).finally(() => {
|
|
278
|
+
if (onProgress) {
|
|
279
|
+
for (const ev of buffered)
|
|
280
|
+
onProgress(ev);
|
|
189
281
|
}
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
return newDb;
|
|
282
|
+
});
|
|
283
|
+
return applied;
|
|
284
|
+
}
|
|
194
285
|
}
|
|
195
|
-
async function
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
if (!db2.objectStoreNames.contains(METADATA_STORE_NAME)) {
|
|
205
|
-
db2.createObjectStore(METADATA_STORE_NAME, { keyPath: "tableName" });
|
|
206
|
-
}
|
|
207
|
-
const store = db2.createObjectStore(tableName, { keyPath: primaryKey, autoIncrement });
|
|
208
|
-
for (const idx of expectedIndexes) {
|
|
209
|
-
store.createIndex(idx.name, idx.keyPath, idx.options);
|
|
210
|
-
}
|
|
211
|
-
});
|
|
212
|
-
const snapshot = {
|
|
213
|
-
version: db.version,
|
|
214
|
-
primaryKey,
|
|
215
|
-
indexes: expectedIndexes,
|
|
216
|
-
recordCount: 0,
|
|
217
|
-
timestamp: Date.now()
|
|
218
|
-
};
|
|
219
|
-
await saveSchemaMetadata(db, tableName, snapshot);
|
|
220
|
-
options.onMigrationProgress?.(`Database created successfully`, 1);
|
|
221
|
-
return db;
|
|
286
|
+
async function runIndexedDbMigrationGroups(groups, options = {}) {
|
|
287
|
+
const { idb, onProgress } = options;
|
|
288
|
+
const all = [];
|
|
289
|
+
for (const group of groups) {
|
|
290
|
+
const runner = idb ? new IndexedDbMigrationRunner(group.dbName, idb) : new IndexedDbMigrationRunner(group.dbName);
|
|
291
|
+
const applied = await runner.run(group.migrations, { onProgress });
|
|
292
|
+
all.push(...applied);
|
|
293
|
+
}
|
|
294
|
+
return all;
|
|
222
295
|
}
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
if (!db2.objectStoreNames.contains(METADATA_STORE_NAME)) {
|
|
246
|
-
db2.createObjectStore(METADATA_STORE_NAME, { keyPath: "tableName" });
|
|
247
|
-
}
|
|
248
|
-
const store2 = db2.createObjectStore(tableName, { keyPath: primaryKey, autoIncrement });
|
|
249
|
-
for (const idx of expectedIndexes) {
|
|
250
|
-
store2.createIndex(idx.name, idx.keyPath, idx.options);
|
|
251
|
-
}
|
|
252
|
-
});
|
|
253
|
-
const snapshot2 = {
|
|
254
|
-
version: db.version,
|
|
255
|
-
primaryKey,
|
|
256
|
-
indexes: expectedIndexes,
|
|
257
|
-
recordCount: 0,
|
|
258
|
-
timestamp: Date.now()
|
|
259
|
-
};
|
|
260
|
-
await saveSchemaMetadata(db, tableName, snapshot2);
|
|
261
|
-
options.onMigrationProgress?.(`Database created successfully`, 1);
|
|
262
|
-
return db;
|
|
263
|
-
}
|
|
264
|
-
if (!db.objectStoreNames.contains(METADATA_STORE_NAME)) {
|
|
265
|
-
const currentVersion = db.version;
|
|
266
|
-
db.close();
|
|
267
|
-
db = await openIndexedDbTable(tableName, currentVersion + 1, (event) => {
|
|
268
|
-
const db2 = event.target.result;
|
|
269
|
-
if (!db2.objectStoreNames.contains(METADATA_STORE_NAME)) {
|
|
270
|
-
db2.createObjectStore(METADATA_STORE_NAME, { keyPath: "tableName" });
|
|
296
|
+
|
|
297
|
+
// src/migrations/indexedDbQueueMigrations.ts
|
|
298
|
+
function indexedDbQueueMigrations(tableName, prefixes) {
|
|
299
|
+
const component = `queue:indexeddb:${tableName}`;
|
|
300
|
+
const prefixCols = prefixes.map((p) => p.name);
|
|
301
|
+
const k = (cols) => [...prefixCols, ...cols];
|
|
302
|
+
return [
|
|
303
|
+
{
|
|
304
|
+
component,
|
|
305
|
+
version: 1,
|
|
306
|
+
description: "Create queue object store + indexes",
|
|
307
|
+
up({ db }) {
|
|
308
|
+
if (!db.objectStoreNames.contains(tableName)) {
|
|
309
|
+
const store = db.createObjectStore(tableName, { keyPath: "id" });
|
|
310
|
+
store.createIndex("queue_status", k(["queue", "status"]), { unique: false });
|
|
311
|
+
store.createIndex("queue_status_run_after", k(["queue", "status", "run_after"]), {
|
|
312
|
+
unique: false
|
|
313
|
+
});
|
|
314
|
+
store.createIndex("queue_job_run_id", k(["queue", "job_run_id"]), { unique: false });
|
|
315
|
+
store.createIndex("queue_fingerprint_status", k(["queue", "fingerprint", "status"]), {
|
|
316
|
+
unique: false
|
|
317
|
+
});
|
|
271
318
|
}
|
|
272
|
-
}
|
|
273
|
-
}
|
|
274
|
-
if (!db.objectStoreNames.contains(tableName)) {
|
|
275
|
-
options.onMigrationProgress?.(`Object store ${tableName} does not exist, creating...`, 0);
|
|
276
|
-
db.close();
|
|
277
|
-
return await createNewDatabase(tableName, primaryKey, expectedIndexes, options, autoIncrement);
|
|
278
|
-
}
|
|
279
|
-
const transaction = db.transaction(tableName, "readonly");
|
|
280
|
-
const store = transaction.objectStore(tableName);
|
|
281
|
-
const diff = compareSchemas(store, primaryKey, expectedIndexes);
|
|
282
|
-
await new Promise((resolve) => {
|
|
283
|
-
transaction.oncomplete = () => resolve();
|
|
284
|
-
transaction.onerror = () => resolve();
|
|
285
|
-
});
|
|
286
|
-
const needsMigration = diff.indexesToAdd.length > 0 || diff.indexesToRemove.length > 0 || diff.indexesToModify.length > 0 || diff.needsObjectStoreRecreation;
|
|
287
|
-
if (!needsMigration) {
|
|
288
|
-
options.onMigrationProgress?.(`Schema for ${tableName} is up to date`, 1);
|
|
289
|
-
const snapshot2 = {
|
|
290
|
-
version: db.version,
|
|
291
|
-
primaryKey,
|
|
292
|
-
indexes: expectedIndexes,
|
|
293
|
-
timestamp: Date.now()
|
|
294
|
-
};
|
|
295
|
-
await saveSchemaMetadata(db, tableName, snapshot2);
|
|
296
|
-
return db;
|
|
297
|
-
}
|
|
298
|
-
if (diff.needsObjectStoreRecreation) {
|
|
299
|
-
options.onMigrationProgress?.(`Schema change requires object store recreation for ${tableName}`, 0);
|
|
300
|
-
db = await performDestructiveMigration(db, tableName, primaryKey, expectedIndexes, options, autoIncrement);
|
|
301
|
-
} else {
|
|
302
|
-
options.onMigrationProgress?.(`Performing incremental migration for ${tableName}`, 0);
|
|
303
|
-
db = await performIncrementalMigration(db, tableName, diff, options);
|
|
319
|
+
}
|
|
304
320
|
}
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
return db;
|
|
313
|
-
} catch (err) {
|
|
314
|
-
options.onMigrationWarning?.(`Migration failed for ${tableName}: ${err}`, err);
|
|
315
|
-
throw err;
|
|
316
|
-
}
|
|
321
|
+
];
|
|
322
|
+
}
|
|
323
|
+
function indexedDbQueueMigrationGroup(tableName, prefixes) {
|
|
324
|
+
return {
|
|
325
|
+
dbName: tableName,
|
|
326
|
+
migrations: indexedDbQueueMigrations(tableName, prefixes)
|
|
327
|
+
};
|
|
317
328
|
}
|
|
318
329
|
|
|
319
330
|
// src/job-queue/IndexedDbQueueStorage.ts
|
|
@@ -325,14 +336,12 @@ class IndexedDbQueueStorage {
|
|
|
325
336
|
scope = "process";
|
|
326
337
|
db;
|
|
327
338
|
tableName;
|
|
328
|
-
migrationOptions;
|
|
329
339
|
prefixes;
|
|
330
340
|
prefixValues;
|
|
331
341
|
hybridManager = null;
|
|
332
342
|
hybridOptions;
|
|
333
343
|
constructor(queueName, options = {}) {
|
|
334
344
|
this.queueName = queueName;
|
|
335
|
-
this.migrationOptions = options;
|
|
336
345
|
this.prefixes = options.prefixes ?? [];
|
|
337
346
|
this.prefixValues = options.prefixValues ?? {};
|
|
338
347
|
this.hybridOptions = {
|
|
@@ -346,9 +355,6 @@ class IndexedDbQueueStorage {
|
|
|
346
355
|
this.tableName = "jobs";
|
|
347
356
|
}
|
|
348
357
|
}
|
|
349
|
-
getPrefixColumnNames() {
|
|
350
|
-
return this.prefixes.map((p) => p.name);
|
|
351
|
-
}
|
|
352
358
|
matchesPrefixes(job) {
|
|
353
359
|
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
354
360
|
if (job[key] !== value) {
|
|
@@ -363,37 +369,22 @@ class IndexedDbQueueStorage {
|
|
|
363
369
|
async getDb() {
|
|
364
370
|
if (this.db)
|
|
365
371
|
return this.db;
|
|
366
|
-
await this.
|
|
372
|
+
await this.migrate();
|
|
367
373
|
return this.db;
|
|
368
374
|
}
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
keyPath: buildKeyPath(["queue", "status", "run_after"]),
|
|
383
|
-
options: { unique: false }
|
|
384
|
-
},
|
|
385
|
-
{
|
|
386
|
-
name: "queue_job_run_id",
|
|
387
|
-
keyPath: buildKeyPath(["queue", "job_run_id"]),
|
|
388
|
-
options: { unique: false }
|
|
389
|
-
},
|
|
390
|
-
{
|
|
391
|
-
name: "queue_fingerprint_status",
|
|
392
|
-
keyPath: buildKeyPath(["queue", "fingerprint", "status"]),
|
|
393
|
-
options: { unique: false }
|
|
394
|
-
}
|
|
395
|
-
];
|
|
396
|
-
this.db = await ensureIndexedDbTable(this.tableName, "id", expectedIndexes, this.migrationOptions);
|
|
375
|
+
getMigrations() {
|
|
376
|
+
return indexedDbQueueMigrations(this.tableName, this.prefixes);
|
|
377
|
+
}
|
|
378
|
+
async migrate() {
|
|
379
|
+
if (this.db) {
|
|
380
|
+
try {
|
|
381
|
+
this.db.close();
|
|
382
|
+
} catch {}
|
|
383
|
+
this.db = undefined;
|
|
384
|
+
}
|
|
385
|
+
const runner = new IndexedDbMigrationRunner(this.tableName);
|
|
386
|
+
await runner.run(this.getMigrations());
|
|
387
|
+
this.db = await openIdb(this.tableName);
|
|
397
388
|
}
|
|
398
389
|
async add(job) {
|
|
399
390
|
const db = await this.getDb();
|
|
@@ -836,7 +827,7 @@ class IndexedDbQueueStorage {
|
|
|
836
827
|
this.hybridManager = new HybridSubscriptionManager(channelName, async () => {
|
|
837
828
|
const jobs = await this.getAllJobs();
|
|
838
829
|
return new Map(jobs.map((j) => [j.id, j]));
|
|
839
|
-
}, (a, b) =>
|
|
830
|
+
}, (a, b) => deepEqual(a, b), {
|
|
840
831
|
insert: (item) => ({ type: "INSERT", new: item }),
|
|
841
832
|
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
842
833
|
delete: (item) => ({ type: "DELETE", old: item })
|
|
@@ -863,7 +854,7 @@ class IndexedDbQueueStorage {
|
|
|
863
854
|
const old = lastKnownJobs.get(id);
|
|
864
855
|
if (!old) {
|
|
865
856
|
callback({ type: "INSERT", new: job });
|
|
866
|
-
} else if (!
|
|
857
|
+
} else if (!deepEqual(old, job)) {
|
|
867
858
|
callback({ type: "UPDATE", old, new: job });
|
|
868
859
|
}
|
|
869
860
|
}
|
|
@@ -899,6 +890,59 @@ class IndexedDbQueueStorage {
|
|
|
899
890
|
}
|
|
900
891
|
// src/job-queue/IndexedDbRateLimiterStorage.ts
|
|
901
892
|
import { createServiceToken as createServiceToken2 } from "@workglow/util";
|
|
893
|
+
|
|
894
|
+
// src/migrations/indexedDbRateLimiterMigrations.ts
|
|
895
|
+
function indexedDbRateLimiterExecutionMigrations(executionTableName, prefixes) {
|
|
896
|
+
const component = `rate-limiter:indexeddb:${executionTableName}`;
|
|
897
|
+
const prefixCols = prefixes.map((p) => p.name);
|
|
898
|
+
const k = (cols) => [...prefixCols, ...cols];
|
|
899
|
+
return [
|
|
900
|
+
{
|
|
901
|
+
component,
|
|
902
|
+
version: 1,
|
|
903
|
+
description: "Create rate-limiter execution object store + indexes",
|
|
904
|
+
up({ db }) {
|
|
905
|
+
if (!db.objectStoreNames.contains(executionTableName)) {
|
|
906
|
+
const store = db.createObjectStore(executionTableName, { keyPath: "id" });
|
|
907
|
+
store.createIndex("queue_executed_at", k(["queue_name", "executed_at"]), {
|
|
908
|
+
unique: false
|
|
909
|
+
});
|
|
910
|
+
}
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
];
|
|
914
|
+
}
|
|
915
|
+
function indexedDbRateLimiterNextAvailableMigrations(nextAvailableTableName, prefixes) {
|
|
916
|
+
const component = `rate-limiter:indexeddb:${nextAvailableTableName}`;
|
|
917
|
+
const prefixCols = prefixes.map((p) => p.name);
|
|
918
|
+
const keyField = [...prefixCols, "queue_name"].join("_");
|
|
919
|
+
return [
|
|
920
|
+
{
|
|
921
|
+
component,
|
|
922
|
+
version: 1,
|
|
923
|
+
description: "Create rate-limiter next_available object store",
|
|
924
|
+
up({ db }) {
|
|
925
|
+
if (!db.objectStoreNames.contains(nextAvailableTableName)) {
|
|
926
|
+
db.createObjectStore(nextAvailableTableName, { keyPath: keyField });
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
];
|
|
931
|
+
}
|
|
932
|
+
function indexedDbRateLimiterMigrationGroups(executionTableName, nextAvailableTableName, prefixes) {
|
|
933
|
+
return [
|
|
934
|
+
{
|
|
935
|
+
dbName: executionTableName,
|
|
936
|
+
migrations: indexedDbRateLimiterExecutionMigrations(executionTableName, prefixes)
|
|
937
|
+
},
|
|
938
|
+
{
|
|
939
|
+
dbName: nextAvailableTableName,
|
|
940
|
+
migrations: indexedDbRateLimiterNextAvailableMigrations(nextAvailableTableName, prefixes)
|
|
941
|
+
}
|
|
942
|
+
];
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
// src/job-queue/IndexedDbRateLimiterStorage.ts
|
|
902
946
|
var INDEXED_DB_RATE_LIMITER_STORAGE = createServiceToken2("ratelimiter.storage.indexedDb");
|
|
903
947
|
|
|
904
948
|
class IndexedDbRateLimiterStorage {
|
|
@@ -907,11 +951,9 @@ class IndexedDbRateLimiterStorage {
|
|
|
907
951
|
nextAvailableDb;
|
|
908
952
|
executionTableName;
|
|
909
953
|
nextAvailableTableName;
|
|
910
|
-
migrationOptions;
|
|
911
954
|
prefixes;
|
|
912
955
|
prefixValues;
|
|
913
956
|
constructor(options = {}) {
|
|
914
|
-
this.migrationOptions = options;
|
|
915
957
|
this.prefixes = options.prefixes ?? [];
|
|
916
958
|
this.prefixValues = options.prefixValues ?? {};
|
|
917
959
|
if (this.prefixes.length > 0) {
|
|
@@ -940,36 +982,34 @@ class IndexedDbRateLimiterStorage {
|
|
|
940
982
|
async getExecutionDb() {
|
|
941
983
|
if (this.executionDb)
|
|
942
984
|
return this.executionDb;
|
|
943
|
-
await this.
|
|
985
|
+
await this.migrate();
|
|
944
986
|
return this.executionDb;
|
|
945
987
|
}
|
|
946
988
|
async getNextAvailableDb() {
|
|
947
989
|
if (this.nextAvailableDb)
|
|
948
990
|
return this.nextAvailableDb;
|
|
949
|
-
await this.
|
|
991
|
+
await this.migrate();
|
|
950
992
|
return this.nextAvailableDb;
|
|
951
993
|
}
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
];
|
|
972
|
-
this.nextAvailableDb = await ensureIndexedDbTable(this.nextAvailableTableName, buildKeyPath(["queue_name"]).join("_"), nextAvailableIndexes, this.migrationOptions);
|
|
994
|
+
getMigrations() {
|
|
995
|
+
return indexedDbRateLimiterMigrationGroups(this.executionTableName, this.nextAvailableTableName, this.prefixes);
|
|
996
|
+
}
|
|
997
|
+
async migrate() {
|
|
998
|
+
if (this.executionDb) {
|
|
999
|
+
try {
|
|
1000
|
+
this.executionDb.close();
|
|
1001
|
+
} catch {}
|
|
1002
|
+
this.executionDb = undefined;
|
|
1003
|
+
}
|
|
1004
|
+
if (this.nextAvailableDb) {
|
|
1005
|
+
try {
|
|
1006
|
+
this.nextAvailableDb.close();
|
|
1007
|
+
} catch {}
|
|
1008
|
+
this.nextAvailableDb = undefined;
|
|
1009
|
+
}
|
|
1010
|
+
await runIndexedDbMigrationGroups(this.getMigrations());
|
|
1011
|
+
this.executionDb = await openIdb(this.executionTableName);
|
|
1012
|
+
this.nextAvailableDb = await openIdb(this.nextAvailableTableName);
|
|
973
1013
|
}
|
|
974
1014
|
async tryReserveExecution(queueName, maxExecutions, windowMs) {
|
|
975
1015
|
const nextIso = await this.getNextAvailableTime(queueName);
|
|
@@ -1186,10 +1226,18 @@ class IndexedDbRateLimiterStorage {
|
|
|
1186
1226
|
}
|
|
1187
1227
|
}
|
|
1188
1228
|
export {
|
|
1229
|
+
runIndexedDbMigrationGroups,
|
|
1230
|
+
indexedDbRateLimiterNextAvailableMigrations,
|
|
1231
|
+
indexedDbRateLimiterMigrationGroups,
|
|
1232
|
+
indexedDbRateLimiterExecutionMigrations,
|
|
1233
|
+
indexedDbQueueMigrations,
|
|
1234
|
+
indexedDbQueueMigrationGroup,
|
|
1235
|
+
MigrationAbortedByOtherTabError,
|
|
1189
1236
|
IndexedDbRateLimiterStorage,
|
|
1190
1237
|
IndexedDbQueueStorage,
|
|
1238
|
+
IndexedDbMigrationRunner,
|
|
1191
1239
|
INDEXED_DB_RATE_LIMITER_STORAGE,
|
|
1192
1240
|
INDEXED_DB_QUEUE_STORAGE
|
|
1193
1241
|
};
|
|
1194
1242
|
|
|
1195
|
-
//# debugId=
|
|
1243
|
+
//# debugId=676DC1BE327F4D4964756E2164756E21
|