convex-batch-processor 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +337 -0
- package/dist/client/index.d.ts +194 -0
- package/dist/client/index.d.ts.map +1 -0
- package/dist/client/index.js +75 -0
- package/dist/client/index.js.map +1 -0
- package/dist/component/_generated/api.d.ts +34 -0
- package/dist/component/_generated/api.d.ts.map +1 -0
- package/dist/component/_generated/api.js +31 -0
- package/dist/component/_generated/api.js.map +1 -0
- package/dist/component/_generated/component.d.ts +77 -0
- package/dist/component/_generated/component.d.ts.map +1 -0
- package/dist/component/_generated/component.js +11 -0
- package/dist/component/_generated/component.js.map +1 -0
- package/dist/component/_generated/dataModel.d.ts +46 -0
- package/dist/component/_generated/dataModel.d.ts.map +1 -0
- package/dist/component/_generated/dataModel.js +11 -0
- package/dist/component/_generated/dataModel.js.map +1 -0
- package/dist/component/_generated/server.d.ts +121 -0
- package/dist/component/_generated/server.d.ts.map +1 -0
- package/dist/component/_generated/server.js +78 -0
- package/dist/component/_generated/server.js.map +1 -0
- package/dist/component/convex.config.d.ts +3 -0
- package/dist/component/convex.config.d.ts.map +1 -0
- package/dist/component/convex.config.js +3 -0
- package/dist/component/convex.config.js.map +1 -0
- package/dist/component/lib.d.ts +261 -0
- package/dist/component/lib.d.ts.map +1 -0
- package/dist/component/lib.js +629 -0
- package/dist/component/lib.js.map +1 -0
- package/dist/component/schema.d.ts +100 -0
- package/dist/component/schema.d.ts.map +1 -0
- package/dist/component/schema.js +49 -0
- package/dist/component/schema.js.map +1 -0
- package/package.json +63 -0
- package/src/client/index.test.ts +121 -0
- package/src/client/index.ts +308 -0
- package/src/component/_generated/api.ts +50 -0
- package/src/component/_generated/component.ts +133 -0
- package/src/component/_generated/dataModel.ts +60 -0
- package/src/component/_generated/server.ts +156 -0
- package/src/component/convex.config.ts +3 -0
- package/src/component/lib.ts +792 -0
- package/src/component/schema.ts +57 -0
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
declare const _default: import("convex/server").SchemaDefinition<{
|
|
2
|
+
batches: import("convex/server").TableDefinition<import("convex/values").VObject<{
|
|
3
|
+
scheduledFlushId?: import("convex/values").GenericId<"_scheduled_functions"> | undefined;
|
|
4
|
+
items: any[];
|
|
5
|
+
batchId: string;
|
|
6
|
+
config: {
|
|
7
|
+
maxBatchSize: number;
|
|
8
|
+
flushIntervalMs: number;
|
|
9
|
+
processBatchHandle: string;
|
|
10
|
+
};
|
|
11
|
+
status: "accumulating" | "flushing" | "completed";
|
|
12
|
+
itemCount: number;
|
|
13
|
+
createdAt: number;
|
|
14
|
+
lastUpdatedAt: number;
|
|
15
|
+
}, {
|
|
16
|
+
batchId: import("convex/values").VString<string, "required">;
|
|
17
|
+
items: import("convex/values").VArray<any[], import("convex/values").VAny<any, "required", string>, "required">;
|
|
18
|
+
itemCount: import("convex/values").VFloat64<number, "required">;
|
|
19
|
+
createdAt: import("convex/values").VFloat64<number, "required">;
|
|
20
|
+
lastUpdatedAt: import("convex/values").VFloat64<number, "required">;
|
|
21
|
+
status: import("convex/values").VUnion<"accumulating" | "flushing" | "completed", [import("convex/values").VLiteral<"accumulating", "required">, import("convex/values").VLiteral<"flushing", "required">, import("convex/values").VLiteral<"completed", "required">], "required", never>;
|
|
22
|
+
config: import("convex/values").VObject<{
|
|
23
|
+
maxBatchSize: number;
|
|
24
|
+
flushIntervalMs: number;
|
|
25
|
+
processBatchHandle: string;
|
|
26
|
+
}, {
|
|
27
|
+
maxBatchSize: import("convex/values").VFloat64<number, "required">;
|
|
28
|
+
flushIntervalMs: import("convex/values").VFloat64<number, "required">;
|
|
29
|
+
processBatchHandle: import("convex/values").VString<string, "required">;
|
|
30
|
+
}, "required", "maxBatchSize" | "flushIntervalMs" | "processBatchHandle">;
|
|
31
|
+
scheduledFlushId: import("convex/values").VId<import("convex/values").GenericId<"_scheduled_functions"> | undefined, "optional">;
|
|
32
|
+
}, "required", "items" | "batchId" | "config" | "status" | "itemCount" | "createdAt" | "lastUpdatedAt" | "scheduledFlushId" | "config.maxBatchSize" | "config.flushIntervalMs" | "config.processBatchHandle">, {
|
|
33
|
+
by_batchId: ["batchId", "_creationTime"];
|
|
34
|
+
by_status: ["status", "_creationTime"];
|
|
35
|
+
}, {}, {}>;
|
|
36
|
+
iteratorJobs: import("convex/server").TableDefinition<import("convex/values").VObject<{
|
|
37
|
+
cursor?: string | undefined;
|
|
38
|
+
errorMessage?: string | undefined;
|
|
39
|
+
lastRunAt?: number | undefined;
|
|
40
|
+
jobId: string;
|
|
41
|
+
processedCount: number;
|
|
42
|
+
config: {
|
|
43
|
+
onCompleteHandle?: string | undefined;
|
|
44
|
+
maxRetries?: number | undefined;
|
|
45
|
+
batchSize: number;
|
|
46
|
+
processBatchHandle: string;
|
|
47
|
+
delayBetweenBatchesMs: number;
|
|
48
|
+
getNextBatchHandle: string;
|
|
49
|
+
};
|
|
50
|
+
status: "completed" | "pending" | "running" | "paused" | "failed";
|
|
51
|
+
createdAt: number;
|
|
52
|
+
retryCount: number;
|
|
53
|
+
}, {
|
|
54
|
+
jobId: import("convex/values").VString<string, "required">;
|
|
55
|
+
cursor: import("convex/values").VString<string | undefined, "optional">;
|
|
56
|
+
processedCount: import("convex/values").VFloat64<number, "required">;
|
|
57
|
+
status: import("convex/values").VUnion<"completed" | "pending" | "running" | "paused" | "failed", [import("convex/values").VLiteral<"pending", "required">, import("convex/values").VLiteral<"running", "required">, import("convex/values").VLiteral<"paused", "required">, import("convex/values").VLiteral<"completed", "required">, import("convex/values").VLiteral<"failed", "required">], "required", never>;
|
|
58
|
+
config: import("convex/values").VObject<{
|
|
59
|
+
onCompleteHandle?: string | undefined;
|
|
60
|
+
maxRetries?: number | undefined;
|
|
61
|
+
batchSize: number;
|
|
62
|
+
processBatchHandle: string;
|
|
63
|
+
delayBetweenBatchesMs: number;
|
|
64
|
+
getNextBatchHandle: string;
|
|
65
|
+
}, {
|
|
66
|
+
batchSize: import("convex/values").VFloat64<number, "required">;
|
|
67
|
+
delayBetweenBatchesMs: import("convex/values").VFloat64<number, "required">;
|
|
68
|
+
getNextBatchHandle: import("convex/values").VString<string, "required">;
|
|
69
|
+
processBatchHandle: import("convex/values").VString<string, "required">;
|
|
70
|
+
onCompleteHandle: import("convex/values").VString<string | undefined, "optional">;
|
|
71
|
+
maxRetries: import("convex/values").VFloat64<number | undefined, "optional">;
|
|
72
|
+
}, "required", "batchSize" | "processBatchHandle" | "delayBetweenBatchesMs" | "getNextBatchHandle" | "onCompleteHandle" | "maxRetries">;
|
|
73
|
+
retryCount: import("convex/values").VFloat64<number, "required">;
|
|
74
|
+
errorMessage: import("convex/values").VString<string | undefined, "optional">;
|
|
75
|
+
createdAt: import("convex/values").VFloat64<number, "required">;
|
|
76
|
+
lastRunAt: import("convex/values").VFloat64<number | undefined, "optional">;
|
|
77
|
+
}, "required", "cursor" | "jobId" | "processedCount" | "config" | "status" | "createdAt" | "config.processBatchHandle" | "retryCount" | "errorMessage" | "lastRunAt" | "config.batchSize" | "config.delayBetweenBatchesMs" | "config.getNextBatchHandle" | "config.onCompleteHandle" | "config.maxRetries">, {
|
|
78
|
+
by_jobId: ["jobId", "_creationTime"];
|
|
79
|
+
by_status: ["status", "_creationTime"];
|
|
80
|
+
}, {}, {}>;
|
|
81
|
+
flushHistory: import("convex/server").TableDefinition<import("convex/values").VObject<{
|
|
82
|
+
errorMessage?: string | undefined;
|
|
83
|
+
batchId: string;
|
|
84
|
+
itemCount: number;
|
|
85
|
+
flushedAt: number;
|
|
86
|
+
durationMs: number;
|
|
87
|
+
success: boolean;
|
|
88
|
+
}, {
|
|
89
|
+
batchId: import("convex/values").VString<string, "required">;
|
|
90
|
+
itemCount: import("convex/values").VFloat64<number, "required">;
|
|
91
|
+
flushedAt: import("convex/values").VFloat64<number, "required">;
|
|
92
|
+
durationMs: import("convex/values").VFloat64<number, "required">;
|
|
93
|
+
success: import("convex/values").VBoolean<boolean, "required">;
|
|
94
|
+
errorMessage: import("convex/values").VString<string | undefined, "optional">;
|
|
95
|
+
}, "required", "batchId" | "itemCount" | "errorMessage" | "flushedAt" | "durationMs" | "success">, {
|
|
96
|
+
by_batchId: ["batchId", "_creationTime"];
|
|
97
|
+
}, {}, {}>;
|
|
98
|
+
}, true>;
|
|
99
|
+
export default _default;
|
|
100
|
+
//# sourceMappingURL=schema.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/component/schema.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGA,wBAqDG"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { defineSchema, defineTable } from "convex/server";
|
|
2
|
+
import { v } from "convex/values";
|
|
3
|
+
export default defineSchema({
|
|
4
|
+
batches: defineTable({
|
|
5
|
+
batchId: v.string(),
|
|
6
|
+
items: v.array(v.any()),
|
|
7
|
+
itemCount: v.number(),
|
|
8
|
+
createdAt: v.number(),
|
|
9
|
+
lastUpdatedAt: v.number(),
|
|
10
|
+
status: v.union(v.literal("accumulating"), v.literal("flushing"), v.literal("completed")),
|
|
11
|
+
config: v.object({
|
|
12
|
+
maxBatchSize: v.number(),
|
|
13
|
+
flushIntervalMs: v.number(),
|
|
14
|
+
processBatchHandle: v.string(),
|
|
15
|
+
}),
|
|
16
|
+
scheduledFlushId: v.optional(v.id("_scheduled_functions")),
|
|
17
|
+
})
|
|
18
|
+
.index("by_batchId", ["batchId"])
|
|
19
|
+
.index("by_status", ["status"]),
|
|
20
|
+
iteratorJobs: defineTable({
|
|
21
|
+
jobId: v.string(),
|
|
22
|
+
cursor: v.optional(v.string()),
|
|
23
|
+
processedCount: v.number(),
|
|
24
|
+
status: v.union(v.literal("pending"), v.literal("running"), v.literal("paused"), v.literal("completed"), v.literal("failed")),
|
|
25
|
+
config: v.object({
|
|
26
|
+
batchSize: v.number(),
|
|
27
|
+
delayBetweenBatchesMs: v.number(),
|
|
28
|
+
getNextBatchHandle: v.string(),
|
|
29
|
+
processBatchHandle: v.string(),
|
|
30
|
+
onCompleteHandle: v.optional(v.string()),
|
|
31
|
+
maxRetries: v.optional(v.number()),
|
|
32
|
+
}),
|
|
33
|
+
retryCount: v.number(),
|
|
34
|
+
errorMessage: v.optional(v.string()),
|
|
35
|
+
createdAt: v.number(),
|
|
36
|
+
lastRunAt: v.optional(v.number()),
|
|
37
|
+
})
|
|
38
|
+
.index("by_jobId", ["jobId"])
|
|
39
|
+
.index("by_status", ["status"]),
|
|
40
|
+
flushHistory: defineTable({
|
|
41
|
+
batchId: v.string(),
|
|
42
|
+
itemCount: v.number(),
|
|
43
|
+
flushedAt: v.number(),
|
|
44
|
+
durationMs: v.number(),
|
|
45
|
+
success: v.boolean(),
|
|
46
|
+
errorMessage: v.optional(v.string()),
|
|
47
|
+
}).index("by_batchId", ["batchId"]),
|
|
48
|
+
});
|
|
49
|
+
//# sourceMappingURL=schema.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/component/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAElC,eAAe,YAAY,CAAC;IAC3B,OAAO,EAAE,WAAW,CAAC;QACpB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACvB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;QACzB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACzF,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;YAChB,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE;YACxB,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;YAC3B,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE;SAC9B,CAAC;QACF,gBAAgB,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,sBAAsB,CAAC,CAAC;KAC1D,CAAC;SACA,KAAK,CAAC,YAAY,EAAE,CAAC,SAAS,CAAC,CAAC;SAChC,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC;IAEhC,YAAY,EAAE,WAAW,CAAC;QACzB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;QACjB,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QAC9B,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE;QAC1B,MAAM,EAAE,CAAC,CAAC,KAAK,CACd,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EACpB,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EACpB,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,EACnB,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,EACtB,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CACnB;QACD,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;YAChB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;YACrB,qBAAqB,EAAE,CAAC,CAAC,MAAM,EAAE;YACjC,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE;YAC9B,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE;YAC9B,gBAAgB,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YACxC,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;SAClC,CAAC;QACF,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;QACtB,YAAY,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QACpC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,SAAS,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;KACjC,CAAC;SACA,KAAK,CAAC,UAAU,EAAE,CAAC,OAAO,CAAC,CAAC;SAC5B,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC;IAEhC,YAAY,EAAE,WAAW,CAAC;QACzB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;QACtB,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE;QACpB,YAAY,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;KACpC,CAAC,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,SAAS,CAAC,CAAC;CACnC,CAAC,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "convex-batch-processor",
|
|
3
|
+
"version": "1.0.2",
|
|
4
|
+
"description": "Batch processing component for Convex - batch accumulator and table iterator",
|
|
5
|
+
"repository": {
|
|
6
|
+
"type": "git",
|
|
7
|
+
"url": "https://github.com/blocknavi/convex-batch-processor"
|
|
8
|
+
},
|
|
9
|
+
"homepage": "https://github.com/blocknavi/convex-batch-processor#readme",
|
|
10
|
+
"bugs": {
|
|
11
|
+
"url": "https://github.com/blocknavi/convex-batch-processor/issues"
|
|
12
|
+
},
|
|
13
|
+
"license": "MIT",
|
|
14
|
+
"keywords": ["convex", "component", "batch", "processing", "iterator", "accumulator"],
|
|
15
|
+
"type": "module",
|
|
16
|
+
"scripts": {
|
|
17
|
+
"codegen": "npx convex codegen --component-dir src/component",
|
|
18
|
+
"build": "tsc --project ./tsconfig.build.json",
|
|
19
|
+
"build:clean": "rm -rf dist *.tsbuildinfo && pnpm run build",
|
|
20
|
+
"test": "vitest run",
|
|
21
|
+
"test:watch": "vitest",
|
|
22
|
+
"lint": "biome check .",
|
|
23
|
+
"lint:fix": "biome check --write .",
|
|
24
|
+
"format": "biome format --write .",
|
|
25
|
+
"prepublishOnly": "pnpm run build && pnpm run lint",
|
|
26
|
+
"preversion": "pnpm install && pnpm run build:clean && pnpm run test && pnpm run lint",
|
|
27
|
+
"alpha": "npm version prerelease --preid alpha && npm publish --tag alpha && git push --follow-tags",
|
|
28
|
+
"release": "npm version patch && npm publish && git push --follow-tags"
|
|
29
|
+
},
|
|
30
|
+
"files": ["dist", "src"],
|
|
31
|
+
"exports": {
|
|
32
|
+
"./package.json": "./package.json",
|
|
33
|
+
".": {
|
|
34
|
+
"types": "./dist/client/index.d.ts",
|
|
35
|
+
"default": "./dist/client/index.js"
|
|
36
|
+
},
|
|
37
|
+
"./convex.config.js": {
|
|
38
|
+
"types": "./dist/component/convex.config.d.ts",
|
|
39
|
+
"default": "./dist/component/convex.config.js"
|
|
40
|
+
},
|
|
41
|
+
"./convex.config": {
|
|
42
|
+
"types": "./dist/component/convex.config.d.ts",
|
|
43
|
+
"default": "./dist/component/convex.config.js"
|
|
44
|
+
},
|
|
45
|
+
"./_generated/component.js": {
|
|
46
|
+
"types": "./dist/component/_generated/component.d.ts"
|
|
47
|
+
},
|
|
48
|
+
"./_generated/component": {
|
|
49
|
+
"types": "./dist/component/_generated/component.d.ts"
|
|
50
|
+
}
|
|
51
|
+
},
|
|
52
|
+
"dependencies": {
|
|
53
|
+
"convex": "^1.31.6"
|
|
54
|
+
},
|
|
55
|
+
"devDependencies": {
|
|
56
|
+
"@biomejs/biome": "^1.9.0",
|
|
57
|
+
"typescript": "^5.6.0",
|
|
58
|
+
"vitest": "^2.0.0"
|
|
59
|
+
},
|
|
60
|
+
"peerDependencies": {
|
|
61
|
+
"convex": "^1.31.6"
|
|
62
|
+
}
|
|
63
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { describe, expect, test } from "vitest";
|
|
2
|
+
import {
|
|
3
|
+
type BatchConfig,
|
|
4
|
+
BatchProcessor,
|
|
5
|
+
type GetNextBatchArgs,
|
|
6
|
+
type GetNextBatchResult,
|
|
7
|
+
type IteratorConfig,
|
|
8
|
+
type OnCompleteArgs,
|
|
9
|
+
type ProcessBatchArgs,
|
|
10
|
+
} from "./index";
|
|
11
|
+
|
|
12
|
+
describe("BatchProcessor client", () => {
|
|
13
|
+
test("exports BatchProcessor class", () => {
|
|
14
|
+
expect(BatchProcessor).toBeDefined();
|
|
15
|
+
expect(typeof BatchProcessor).toBe("function");
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
test("BatchProcessor constructor accepts component API without config", () => {
|
|
19
|
+
// Create a mock component API
|
|
20
|
+
const mockComponent = {
|
|
21
|
+
lib: {
|
|
22
|
+
addItems: {} as any,
|
|
23
|
+
flushBatch: {} as any,
|
|
24
|
+
getBatchStatus: {} as any,
|
|
25
|
+
getFlushHistory: {} as any,
|
|
26
|
+
deleteBatch: {} as any,
|
|
27
|
+
startIteratorJob: {} as any,
|
|
28
|
+
pauseIteratorJob: {} as any,
|
|
29
|
+
resumeIteratorJob: {} as any,
|
|
30
|
+
cancelIteratorJob: {} as any,
|
|
31
|
+
getIteratorJobStatus: {} as any,
|
|
32
|
+
listIteratorJobs: {} as any,
|
|
33
|
+
deleteIteratorJob: {} as any,
|
|
34
|
+
},
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
const processor = new BatchProcessor(mockComponent);
|
|
38
|
+
expect(processor).toBeInstanceOf(BatchProcessor);
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
test("BatchProcessor constructor accepts component API with config", () => {
|
|
42
|
+
const mockComponent = {
|
|
43
|
+
lib: {
|
|
44
|
+
addItems: {} as any,
|
|
45
|
+
flushBatch: {} as any,
|
|
46
|
+
getBatchStatus: {} as any,
|
|
47
|
+
getFlushHistory: {} as any,
|
|
48
|
+
deleteBatch: {} as any,
|
|
49
|
+
startIteratorJob: {} as any,
|
|
50
|
+
pauseIteratorJob: {} as any,
|
|
51
|
+
resumeIteratorJob: {} as any,
|
|
52
|
+
cancelIteratorJob: {} as any,
|
|
53
|
+
getIteratorJobStatus: {} as any,
|
|
54
|
+
listIteratorJobs: {} as any,
|
|
55
|
+
deleteIteratorJob: {} as any,
|
|
56
|
+
},
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
const processor = new BatchProcessor(mockComponent, {
|
|
60
|
+
maxBatchSize: 100,
|
|
61
|
+
flushIntervalMs: 30000,
|
|
62
|
+
processBatch: {} as any, // Mock FunctionReference
|
|
63
|
+
});
|
|
64
|
+
expect(processor).toBeInstanceOf(BatchProcessor);
|
|
65
|
+
});
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
describe("type exports", () => {
|
|
69
|
+
test("BatchConfig type is usable", () => {
|
|
70
|
+
const config: BatchConfig = {
|
|
71
|
+
maxBatchSize: 100,
|
|
72
|
+
flushIntervalMs: 30000,
|
|
73
|
+
processBatch: {} as any, // FunctionReference is opaque, use mock
|
|
74
|
+
};
|
|
75
|
+
expect(config.maxBatchSize).toBe(100);
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
test("IteratorConfig type is usable", () => {
|
|
79
|
+
const config: IteratorConfig = {
|
|
80
|
+
batchSize: 50,
|
|
81
|
+
delayBetweenBatchesMs: 100,
|
|
82
|
+
getNextBatch: {} as any, // FunctionReference mock
|
|
83
|
+
processBatch: {} as any, // FunctionReference mock
|
|
84
|
+
onComplete: {} as any, // FunctionReference mock
|
|
85
|
+
maxRetries: 3,
|
|
86
|
+
};
|
|
87
|
+
expect(config.batchSize).toBe(50);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test("GetNextBatchResult type is usable", () => {
|
|
91
|
+
const result: GetNextBatchResult<{ id: number }> = {
|
|
92
|
+
items: [{ id: 1 }, { id: 2 }],
|
|
93
|
+
cursor: "cursor-123",
|
|
94
|
+
done: false,
|
|
95
|
+
};
|
|
96
|
+
expect(result.items.length).toBe(2);
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test("GetNextBatchArgs type is usable", () => {
|
|
100
|
+
const args: GetNextBatchArgs = {
|
|
101
|
+
cursor: "cursor-123",
|
|
102
|
+
batchSize: 100,
|
|
103
|
+
};
|
|
104
|
+
expect(args.batchSize).toBe(100);
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
test("ProcessBatchArgs type is usable", () => {
|
|
108
|
+
const args: ProcessBatchArgs<{ id: number }> = {
|
|
109
|
+
items: [{ id: 1 }],
|
|
110
|
+
};
|
|
111
|
+
expect(args.items.length).toBe(1);
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
test("OnCompleteArgs type is usable", () => {
|
|
115
|
+
const args: OnCompleteArgs = {
|
|
116
|
+
jobId: "job-123",
|
|
117
|
+
processedCount: 1000,
|
|
118
|
+
};
|
|
119
|
+
expect(args.processedCount).toBe(1000);
|
|
120
|
+
});
|
|
121
|
+
});
|
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type FunctionReference,
|
|
3
|
+
type GenericMutationCtx,
|
|
4
|
+
type GenericQueryCtx,
|
|
5
|
+
createFunctionHandle,
|
|
6
|
+
} from "convex/server";
|
|
7
|
+
|
|
8
|
+
export type BatchStatus = "accumulating" | "flushing" | "completed";
|
|
9
|
+
export type JobStatus = "pending" | "running" | "paused" | "completed" | "failed";
|
|
10
|
+
|
|
11
|
+
// User-facing config interfaces (accept FunctionReference)
|
|
12
|
+
export interface BatchConfig<T = unknown> {
|
|
13
|
+
maxBatchSize: number;
|
|
14
|
+
flushIntervalMs: number;
|
|
15
|
+
processBatch: FunctionReference<"action", "internal", { items: T[] }>;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface IteratorConfig<T = unknown> {
|
|
19
|
+
batchSize: number;
|
|
20
|
+
delayBetweenBatchesMs?: number;
|
|
21
|
+
getNextBatch: FunctionReference<
|
|
22
|
+
"query",
|
|
23
|
+
"internal",
|
|
24
|
+
{ cursor: string | undefined; batchSize: number }
|
|
25
|
+
>;
|
|
26
|
+
processBatch: FunctionReference<"action", "internal", { items: T[] }>;
|
|
27
|
+
onComplete?: FunctionReference<"mutation", "internal", { jobId: string; processedCount: number }>;
|
|
28
|
+
maxRetries?: number;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Internal config interfaces (use string handles for component API)
|
|
32
|
+
interface InternalBatchConfig {
|
|
33
|
+
maxBatchSize: number;
|
|
34
|
+
flushIntervalMs: number;
|
|
35
|
+
processBatchHandle: string;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
interface InternalIteratorConfig {
|
|
39
|
+
batchSize: number;
|
|
40
|
+
delayBetweenBatchesMs?: number;
|
|
41
|
+
getNextBatchHandle: string;
|
|
42
|
+
processBatchHandle: string;
|
|
43
|
+
onCompleteHandle?: string;
|
|
44
|
+
maxRetries?: number;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export interface BatchResult {
|
|
48
|
+
batchId: string;
|
|
49
|
+
itemCount: number;
|
|
50
|
+
flushed: boolean;
|
|
51
|
+
status: BatchStatus;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export interface FlushResult {
|
|
55
|
+
batchId: string;
|
|
56
|
+
itemCount: number;
|
|
57
|
+
flushed: boolean;
|
|
58
|
+
status?: BatchStatus;
|
|
59
|
+
reason?: string;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export interface BatchStatusResult {
|
|
63
|
+
batchId: string;
|
|
64
|
+
itemCount: number;
|
|
65
|
+
status: BatchStatus;
|
|
66
|
+
createdAt: number;
|
|
67
|
+
lastUpdatedAt: number;
|
|
68
|
+
config: BatchConfig;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export interface JobResult {
|
|
72
|
+
jobId: string;
|
|
73
|
+
status: JobStatus;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export interface JobStatusResult {
|
|
77
|
+
jobId: string;
|
|
78
|
+
status: JobStatus;
|
|
79
|
+
processedCount: number;
|
|
80
|
+
cursor?: string;
|
|
81
|
+
retryCount: number;
|
|
82
|
+
errorMessage?: string;
|
|
83
|
+
createdAt: number;
|
|
84
|
+
lastRunAt?: number;
|
|
85
|
+
config: {
|
|
86
|
+
batchSize: number;
|
|
87
|
+
delayBetweenBatchesMs: number;
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export interface JobListItem {
|
|
92
|
+
jobId: string;
|
|
93
|
+
status: JobStatus;
|
|
94
|
+
processedCount: number;
|
|
95
|
+
createdAt: number;
|
|
96
|
+
lastRunAt?: number;
|
|
97
|
+
errorMessage?: string;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export interface FlushHistoryItem {
|
|
101
|
+
batchId: string;
|
|
102
|
+
itemCount: number;
|
|
103
|
+
flushedAt: number;
|
|
104
|
+
durationMs: number;
|
|
105
|
+
success: boolean;
|
|
106
|
+
errorMessage?: string;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export interface BatchProcessorAPI {
|
|
110
|
+
lib: {
|
|
111
|
+
addItems: FunctionReference<
|
|
112
|
+
"mutation",
|
|
113
|
+
"internal",
|
|
114
|
+
{ batchId: string; items: unknown[]; config: InternalBatchConfig },
|
|
115
|
+
BatchResult
|
|
116
|
+
>;
|
|
117
|
+
flushBatch: FunctionReference<"mutation", "internal", { batchId: string }, FlushResult>;
|
|
118
|
+
getBatchStatus: FunctionReference<
|
|
119
|
+
"query",
|
|
120
|
+
"internal",
|
|
121
|
+
{ batchId: string },
|
|
122
|
+
BatchStatusResult | null
|
|
123
|
+
>;
|
|
124
|
+
getFlushHistory: FunctionReference<
|
|
125
|
+
"query",
|
|
126
|
+
"internal",
|
|
127
|
+
{ batchId: string; limit?: number },
|
|
128
|
+
FlushHistoryItem[]
|
|
129
|
+
>;
|
|
130
|
+
deleteBatch: FunctionReference<
|
|
131
|
+
"mutation",
|
|
132
|
+
"internal",
|
|
133
|
+
{ batchId: string },
|
|
134
|
+
{ deleted: boolean; reason?: string }
|
|
135
|
+
>;
|
|
136
|
+
startIteratorJob: FunctionReference<
|
|
137
|
+
"mutation",
|
|
138
|
+
"internal",
|
|
139
|
+
{ jobId: string; config: InternalIteratorConfig },
|
|
140
|
+
JobResult
|
|
141
|
+
>;
|
|
142
|
+
pauseIteratorJob: FunctionReference<"mutation", "internal", { jobId: string }, JobResult>;
|
|
143
|
+
resumeIteratorJob: FunctionReference<"mutation", "internal", { jobId: string }, JobResult>;
|
|
144
|
+
cancelIteratorJob: FunctionReference<
|
|
145
|
+
"mutation",
|
|
146
|
+
"internal",
|
|
147
|
+
{ jobId: string },
|
|
148
|
+
JobResult & { reason?: string }
|
|
149
|
+
>;
|
|
150
|
+
getIteratorJobStatus: FunctionReference<
|
|
151
|
+
"query",
|
|
152
|
+
"internal",
|
|
153
|
+
{ jobId: string },
|
|
154
|
+
JobStatusResult | null
|
|
155
|
+
>;
|
|
156
|
+
listIteratorJobs: FunctionReference<
|
|
157
|
+
"query",
|
|
158
|
+
"internal",
|
|
159
|
+
{ status?: JobStatus; limit?: number },
|
|
160
|
+
JobListItem[]
|
|
161
|
+
>;
|
|
162
|
+
deleteIteratorJob: FunctionReference<
|
|
163
|
+
"mutation",
|
|
164
|
+
"internal",
|
|
165
|
+
{ jobId: string },
|
|
166
|
+
{ deleted: boolean; reason?: string }
|
|
167
|
+
>;
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
export class BatchProcessor<T = unknown> {
|
|
172
|
+
private component: BatchProcessorAPI;
|
|
173
|
+
private config?: BatchConfig<T>;
|
|
174
|
+
private processBatchHandle: string | null = null;
|
|
175
|
+
|
|
176
|
+
constructor(component: BatchProcessorAPI, config?: BatchConfig<T>) {
|
|
177
|
+
this.component = component;
|
|
178
|
+
this.config = config;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
async addItems(ctx: GenericMutationCtx<any>, batchId: string, items: T[]): Promise<BatchResult> {
|
|
182
|
+
if (!this.config) {
|
|
183
|
+
throw new Error(
|
|
184
|
+
"BatchProcessor config with processBatch is required to use addItems. Pass config to the constructor.",
|
|
185
|
+
);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
if (!this.processBatchHandle) {
|
|
189
|
+
this.processBatchHandle = await createFunctionHandle(this.config.processBatch);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
const internalConfig: InternalBatchConfig = {
|
|
193
|
+
maxBatchSize: this.config.maxBatchSize,
|
|
194
|
+
flushIntervalMs: this.config.flushIntervalMs,
|
|
195
|
+
processBatchHandle: this.processBatchHandle,
|
|
196
|
+
};
|
|
197
|
+
|
|
198
|
+
return await ctx.runMutation(this.component.lib.addItems, {
|
|
199
|
+
batchId,
|
|
200
|
+
items,
|
|
201
|
+
config: internalConfig,
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
async flush(ctx: GenericMutationCtx<any>, batchId: string): Promise<FlushResult> {
|
|
206
|
+
return await ctx.runMutation(this.component.lib.flushBatch, { batchId });
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
async getBatchStatus(
|
|
210
|
+
ctx: GenericQueryCtx<any>,
|
|
211
|
+
batchId: string,
|
|
212
|
+
): Promise<BatchStatusResult | null> {
|
|
213
|
+
return await ctx.runQuery(this.component.lib.getBatchStatus, { batchId });
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
async getFlushHistory(
|
|
217
|
+
ctx: GenericQueryCtx<any>,
|
|
218
|
+
batchId: string,
|
|
219
|
+
limit?: number,
|
|
220
|
+
): Promise<FlushHistoryItem[]> {
|
|
221
|
+
return await ctx.runQuery(this.component.lib.getFlushHistory, { batchId, limit });
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
async deleteBatch(
|
|
225
|
+
ctx: GenericMutationCtx<any>,
|
|
226
|
+
batchId: string,
|
|
227
|
+
): Promise<{ deleted: boolean; reason?: string }> {
|
|
228
|
+
return await ctx.runMutation(this.component.lib.deleteBatch, { batchId });
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
async startIterator<T>(
|
|
232
|
+
ctx: GenericMutationCtx<any>,
|
|
233
|
+
jobId: string,
|
|
234
|
+
config: IteratorConfig<T>,
|
|
235
|
+
): Promise<JobResult> {
|
|
236
|
+
const internalConfig: InternalIteratorConfig = {
|
|
237
|
+
batchSize: config.batchSize,
|
|
238
|
+
delayBetweenBatchesMs: config.delayBetweenBatchesMs,
|
|
239
|
+
getNextBatchHandle: await createFunctionHandle(config.getNextBatch),
|
|
240
|
+
processBatchHandle: await createFunctionHandle(config.processBatch),
|
|
241
|
+
onCompleteHandle: config.onComplete
|
|
242
|
+
? await createFunctionHandle(config.onComplete)
|
|
243
|
+
: undefined,
|
|
244
|
+
maxRetries: config.maxRetries,
|
|
245
|
+
};
|
|
246
|
+
|
|
247
|
+
return await ctx.runMutation(this.component.lib.startIteratorJob, {
|
|
248
|
+
jobId,
|
|
249
|
+
config: internalConfig,
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
async pauseIterator(ctx: GenericMutationCtx<any>, jobId: string): Promise<JobResult> {
|
|
254
|
+
return await ctx.runMutation(this.component.lib.pauseIteratorJob, { jobId });
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
async resumeIterator(ctx: GenericMutationCtx<any>, jobId: string): Promise<JobResult> {
|
|
258
|
+
return await ctx.runMutation(this.component.lib.resumeIteratorJob, { jobId });
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
async cancelIterator(
|
|
262
|
+
ctx: GenericMutationCtx<any>,
|
|
263
|
+
jobId: string,
|
|
264
|
+
): Promise<JobResult & { reason?: string }> {
|
|
265
|
+
return await ctx.runMutation(this.component.lib.cancelIteratorJob, { jobId });
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
async getIteratorStatus(
|
|
269
|
+
ctx: GenericQueryCtx<any>,
|
|
270
|
+
jobId: string,
|
|
271
|
+
): Promise<JobStatusResult | null> {
|
|
272
|
+
return await ctx.runQuery(this.component.lib.getIteratorJobStatus, { jobId });
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
async listIteratorJobs(
|
|
276
|
+
ctx: GenericQueryCtx<any>,
|
|
277
|
+
options?: { status?: JobStatus; limit?: number },
|
|
278
|
+
): Promise<JobListItem[]> {
|
|
279
|
+
return await ctx.runQuery(this.component.lib.listIteratorJobs, options ?? {});
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
async deleteIteratorJob(
|
|
283
|
+
ctx: GenericMutationCtx<any>,
|
|
284
|
+
jobId: string,
|
|
285
|
+
): Promise<{ deleted: boolean; reason?: string }> {
|
|
286
|
+
return await ctx.runMutation(this.component.lib.deleteIteratorJob, { jobId });
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
export interface GetNextBatchResult<T = unknown> {
|
|
291
|
+
items: T[];
|
|
292
|
+
cursor: string | undefined;
|
|
293
|
+
done: boolean;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
export interface GetNextBatchArgs {
|
|
297
|
+
cursor: string | undefined;
|
|
298
|
+
batchSize: number;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
export interface ProcessBatchArgs<T = unknown> {
|
|
302
|
+
items: T[];
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
export interface OnCompleteArgs {
|
|
306
|
+
jobId: string;
|
|
307
|
+
processedCount: number;
|
|
308
|
+
}
|