convex-batch-processor 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +337 -0
  3. package/dist/client/index.d.ts +194 -0
  4. package/dist/client/index.d.ts.map +1 -0
  5. package/dist/client/index.js +75 -0
  6. package/dist/client/index.js.map +1 -0
  7. package/dist/component/_generated/api.d.ts +34 -0
  8. package/dist/component/_generated/api.d.ts.map +1 -0
  9. package/dist/component/_generated/api.js +31 -0
  10. package/dist/component/_generated/api.js.map +1 -0
  11. package/dist/component/_generated/component.d.ts +77 -0
  12. package/dist/component/_generated/component.d.ts.map +1 -0
  13. package/dist/component/_generated/component.js +11 -0
  14. package/dist/component/_generated/component.js.map +1 -0
  15. package/dist/component/_generated/dataModel.d.ts +46 -0
  16. package/dist/component/_generated/dataModel.d.ts.map +1 -0
  17. package/dist/component/_generated/dataModel.js +11 -0
  18. package/dist/component/_generated/dataModel.js.map +1 -0
  19. package/dist/component/_generated/server.d.ts +121 -0
  20. package/dist/component/_generated/server.d.ts.map +1 -0
  21. package/dist/component/_generated/server.js +78 -0
  22. package/dist/component/_generated/server.js.map +1 -0
  23. package/dist/component/convex.config.d.ts +3 -0
  24. package/dist/component/convex.config.d.ts.map +1 -0
  25. package/dist/component/convex.config.js +3 -0
  26. package/dist/component/convex.config.js.map +1 -0
  27. package/dist/component/lib.d.ts +261 -0
  28. package/dist/component/lib.d.ts.map +1 -0
  29. package/dist/component/lib.js +629 -0
  30. package/dist/component/lib.js.map +1 -0
  31. package/dist/component/schema.d.ts +100 -0
  32. package/dist/component/schema.d.ts.map +1 -0
  33. package/dist/component/schema.js +49 -0
  34. package/dist/component/schema.js.map +1 -0
  35. package/package.json +63 -0
  36. package/src/client/index.test.ts +121 -0
  37. package/src/client/index.ts +308 -0
  38. package/src/component/_generated/api.ts +50 -0
  39. package/src/component/_generated/component.ts +133 -0
  40. package/src/component/_generated/dataModel.ts +60 -0
  41. package/src/component/_generated/server.ts +156 -0
  42. package/src/component/convex.config.ts +3 -0
  43. package/src/component/lib.ts +792 -0
  44. package/src/component/schema.ts +57 -0
@@ -0,0 +1,100 @@
1
+ declare const _default: import("convex/server").SchemaDefinition<{
2
+ batches: import("convex/server").TableDefinition<import("convex/values").VObject<{
3
+ scheduledFlushId?: import("convex/values").GenericId<"_scheduled_functions"> | undefined;
4
+ items: any[];
5
+ batchId: string;
6
+ config: {
7
+ maxBatchSize: number;
8
+ flushIntervalMs: number;
9
+ processBatchHandle: string;
10
+ };
11
+ status: "accumulating" | "flushing" | "completed";
12
+ itemCount: number;
13
+ createdAt: number;
14
+ lastUpdatedAt: number;
15
+ }, {
16
+ batchId: import("convex/values").VString<string, "required">;
17
+ items: import("convex/values").VArray<any[], import("convex/values").VAny<any, "required", string>, "required">;
18
+ itemCount: import("convex/values").VFloat64<number, "required">;
19
+ createdAt: import("convex/values").VFloat64<number, "required">;
20
+ lastUpdatedAt: import("convex/values").VFloat64<number, "required">;
21
+ status: import("convex/values").VUnion<"accumulating" | "flushing" | "completed", [import("convex/values").VLiteral<"accumulating", "required">, import("convex/values").VLiteral<"flushing", "required">, import("convex/values").VLiteral<"completed", "required">], "required", never>;
22
+ config: import("convex/values").VObject<{
23
+ maxBatchSize: number;
24
+ flushIntervalMs: number;
25
+ processBatchHandle: string;
26
+ }, {
27
+ maxBatchSize: import("convex/values").VFloat64<number, "required">;
28
+ flushIntervalMs: import("convex/values").VFloat64<number, "required">;
29
+ processBatchHandle: import("convex/values").VString<string, "required">;
30
+ }, "required", "maxBatchSize" | "flushIntervalMs" | "processBatchHandle">;
31
+ scheduledFlushId: import("convex/values").VId<import("convex/values").GenericId<"_scheduled_functions"> | undefined, "optional">;
32
+ }, "required", "items" | "batchId" | "config" | "status" | "itemCount" | "createdAt" | "lastUpdatedAt" | "scheduledFlushId" | "config.maxBatchSize" | "config.flushIntervalMs" | "config.processBatchHandle">, {
33
+ by_batchId: ["batchId", "_creationTime"];
34
+ by_status: ["status", "_creationTime"];
35
+ }, {}, {}>;
36
+ iteratorJobs: import("convex/server").TableDefinition<import("convex/values").VObject<{
37
+ cursor?: string | undefined;
38
+ errorMessage?: string | undefined;
39
+ lastRunAt?: number | undefined;
40
+ jobId: string;
41
+ processedCount: number;
42
+ config: {
43
+ onCompleteHandle?: string | undefined;
44
+ maxRetries?: number | undefined;
45
+ batchSize: number;
46
+ processBatchHandle: string;
47
+ delayBetweenBatchesMs: number;
48
+ getNextBatchHandle: string;
49
+ };
50
+ status: "completed" | "pending" | "running" | "paused" | "failed";
51
+ createdAt: number;
52
+ retryCount: number;
53
+ }, {
54
+ jobId: import("convex/values").VString<string, "required">;
55
+ cursor: import("convex/values").VString<string | undefined, "optional">;
56
+ processedCount: import("convex/values").VFloat64<number, "required">;
57
+ status: import("convex/values").VUnion<"completed" | "pending" | "running" | "paused" | "failed", [import("convex/values").VLiteral<"pending", "required">, import("convex/values").VLiteral<"running", "required">, import("convex/values").VLiteral<"paused", "required">, import("convex/values").VLiteral<"completed", "required">, import("convex/values").VLiteral<"failed", "required">], "required", never>;
58
+ config: import("convex/values").VObject<{
59
+ onCompleteHandle?: string | undefined;
60
+ maxRetries?: number | undefined;
61
+ batchSize: number;
62
+ processBatchHandle: string;
63
+ delayBetweenBatchesMs: number;
64
+ getNextBatchHandle: string;
65
+ }, {
66
+ batchSize: import("convex/values").VFloat64<number, "required">;
67
+ delayBetweenBatchesMs: import("convex/values").VFloat64<number, "required">;
68
+ getNextBatchHandle: import("convex/values").VString<string, "required">;
69
+ processBatchHandle: import("convex/values").VString<string, "required">;
70
+ onCompleteHandle: import("convex/values").VString<string | undefined, "optional">;
71
+ maxRetries: import("convex/values").VFloat64<number | undefined, "optional">;
72
+ }, "required", "batchSize" | "processBatchHandle" | "delayBetweenBatchesMs" | "getNextBatchHandle" | "onCompleteHandle" | "maxRetries">;
73
+ retryCount: import("convex/values").VFloat64<number, "required">;
74
+ errorMessage: import("convex/values").VString<string | undefined, "optional">;
75
+ createdAt: import("convex/values").VFloat64<number, "required">;
76
+ lastRunAt: import("convex/values").VFloat64<number | undefined, "optional">;
77
+ }, "required", "cursor" | "jobId" | "processedCount" | "config" | "status" | "createdAt" | "config.processBatchHandle" | "retryCount" | "errorMessage" | "lastRunAt" | "config.batchSize" | "config.delayBetweenBatchesMs" | "config.getNextBatchHandle" | "config.onCompleteHandle" | "config.maxRetries">, {
78
+ by_jobId: ["jobId", "_creationTime"];
79
+ by_status: ["status", "_creationTime"];
80
+ }, {}, {}>;
81
+ flushHistory: import("convex/server").TableDefinition<import("convex/values").VObject<{
82
+ errorMessage?: string | undefined;
83
+ batchId: string;
84
+ itemCount: number;
85
+ flushedAt: number;
86
+ durationMs: number;
87
+ success: boolean;
88
+ }, {
89
+ batchId: import("convex/values").VString<string, "required">;
90
+ itemCount: import("convex/values").VFloat64<number, "required">;
91
+ flushedAt: import("convex/values").VFloat64<number, "required">;
92
+ durationMs: import("convex/values").VFloat64<number, "required">;
93
+ success: import("convex/values").VBoolean<boolean, "required">;
94
+ errorMessage: import("convex/values").VString<string | undefined, "optional">;
95
+ }, "required", "batchId" | "itemCount" | "errorMessage" | "flushedAt" | "durationMs" | "success">, {
96
+ by_batchId: ["batchId", "_creationTime"];
97
+ }, {}, {}>;
98
+ }, true>;
99
+ export default _default;
100
+ //# sourceMappingURL=schema.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/component/schema.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGA,wBAqDG"}
@@ -0,0 +1,49 @@
1
+ import { defineSchema, defineTable } from "convex/server";
2
+ import { v } from "convex/values";
3
+ export default defineSchema({
4
+ batches: defineTable({
5
+ batchId: v.string(),
6
+ items: v.array(v.any()),
7
+ itemCount: v.number(),
8
+ createdAt: v.number(),
9
+ lastUpdatedAt: v.number(),
10
+ status: v.union(v.literal("accumulating"), v.literal("flushing"), v.literal("completed")),
11
+ config: v.object({
12
+ maxBatchSize: v.number(),
13
+ flushIntervalMs: v.number(),
14
+ processBatchHandle: v.string(),
15
+ }),
16
+ scheduledFlushId: v.optional(v.id("_scheduled_functions")),
17
+ })
18
+ .index("by_batchId", ["batchId"])
19
+ .index("by_status", ["status"]),
20
+ iteratorJobs: defineTable({
21
+ jobId: v.string(),
22
+ cursor: v.optional(v.string()),
23
+ processedCount: v.number(),
24
+ status: v.union(v.literal("pending"), v.literal("running"), v.literal("paused"), v.literal("completed"), v.literal("failed")),
25
+ config: v.object({
26
+ batchSize: v.number(),
27
+ delayBetweenBatchesMs: v.number(),
28
+ getNextBatchHandle: v.string(),
29
+ processBatchHandle: v.string(),
30
+ onCompleteHandle: v.optional(v.string()),
31
+ maxRetries: v.optional(v.number()),
32
+ }),
33
+ retryCount: v.number(),
34
+ errorMessage: v.optional(v.string()),
35
+ createdAt: v.number(),
36
+ lastRunAt: v.optional(v.number()),
37
+ })
38
+ .index("by_jobId", ["jobId"])
39
+ .index("by_status", ["status"]),
40
+ flushHistory: defineTable({
41
+ batchId: v.string(),
42
+ itemCount: v.number(),
43
+ flushedAt: v.number(),
44
+ durationMs: v.number(),
45
+ success: v.boolean(),
46
+ errorMessage: v.optional(v.string()),
47
+ }).index("by_batchId", ["batchId"]),
48
+ });
49
+ //# sourceMappingURL=schema.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/component/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAElC,eAAe,YAAY,CAAC;IAC3B,OAAO,EAAE,WAAW,CAAC;QACpB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACvB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;QACzB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACzF,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;YAChB,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE;YACxB,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;YAC3B,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE;SAC9B,CAAC;QACF,gBAAgB,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,sBAAsB,CAAC,CAAC;KAC1D,CAAC;SACA,KAAK,CAAC,YAAY,EAAE,CAAC,SAAS,CAAC,CAAC;SAChC,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC;IAEhC,YAAY,EAAE,WAAW,CAAC;QACzB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;QACjB,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QAC9B,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE;QAC1B,MAAM,EAAE,CAAC,CAAC,KAAK,CACd,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EACpB,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EACpB,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,EACnB,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,EACtB,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CACnB;QACD,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;YAChB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;YACrB,qBAAqB,EAAE,CAAC,CAAC,MAAM,EAAE;YACjC,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE;YAC9B,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE;YAC9B,gBAAgB,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YACxC,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;SAClC,CAAC;QACF,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;QACtB,YAAY,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QACpC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,SAAS,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;KACjC,CAAC;SACA,KAAK,CAAC,UAAU,EAAE,CAAC,OAAO,CAAC,CAAC;SAC5B,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC;IAEhC,YAAY,EAAE,WAAW,CAAC;QACzB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;QACtB,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE;QACpB,YAAY,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;KACpC,CAAC,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,SAAS,CAAC,CAAC;CACnC,CAAC,CAAC"}
package/package.json ADDED
@@ -0,0 +1,63 @@
1
+ {
2
+ "name": "convex-batch-processor",
3
+ "version": "1.0.2",
4
+ "description": "Batch processing component for Convex - batch accumulator and table iterator",
5
+ "repository": {
6
+ "type": "git",
7
+ "url": "https://github.com/blocknavi/convex-batch-processor"
8
+ },
9
+ "homepage": "https://github.com/blocknavi/convex-batch-processor#readme",
10
+ "bugs": {
11
+ "url": "https://github.com/blocknavi/convex-batch-processor/issues"
12
+ },
13
+ "license": "MIT",
14
+ "keywords": ["convex", "component", "batch", "processing", "iterator", "accumulator"],
15
+ "type": "module",
16
+ "scripts": {
17
+ "codegen": "npx convex codegen --component-dir src/component",
18
+ "build": "tsc --project ./tsconfig.build.json",
19
+ "build:clean": "rm -rf dist *.tsbuildinfo && pnpm run build",
20
+ "test": "vitest run",
21
+ "test:watch": "vitest",
22
+ "lint": "biome check .",
23
+ "lint:fix": "biome check --write .",
24
+ "format": "biome format --write .",
25
+ "prepublishOnly": "pnpm run build && pnpm run lint",
26
+ "preversion": "pnpm install && pnpm run build:clean && pnpm run test && pnpm run lint",
27
+ "alpha": "npm version prerelease --preid alpha && npm publish --tag alpha && git push --follow-tags",
28
+ "release": "npm version patch && npm publish && git push --follow-tags"
29
+ },
30
+ "files": ["dist", "src"],
31
+ "exports": {
32
+ "./package.json": "./package.json",
33
+ ".": {
34
+ "types": "./dist/client/index.d.ts",
35
+ "default": "./dist/client/index.js"
36
+ },
37
+ "./convex.config.js": {
38
+ "types": "./dist/component/convex.config.d.ts",
39
+ "default": "./dist/component/convex.config.js"
40
+ },
41
+ "./convex.config": {
42
+ "types": "./dist/component/convex.config.d.ts",
43
+ "default": "./dist/component/convex.config.js"
44
+ },
45
+ "./_generated/component.js": {
46
+ "types": "./dist/component/_generated/component.d.ts"
47
+ },
48
+ "./_generated/component": {
49
+ "types": "./dist/component/_generated/component.d.ts"
50
+ }
51
+ },
52
+ "dependencies": {
53
+ "convex": "^1.31.6"
54
+ },
55
+ "devDependencies": {
56
+ "@biomejs/biome": "^1.9.0",
57
+ "typescript": "^5.6.0",
58
+ "vitest": "^2.0.0"
59
+ },
60
+ "peerDependencies": {
61
+ "convex": "^1.31.6"
62
+ }
63
+ }
@@ -0,0 +1,121 @@
1
+ import { describe, expect, test } from "vitest";
2
+ import {
3
+ type BatchConfig,
4
+ BatchProcessor,
5
+ type GetNextBatchArgs,
6
+ type GetNextBatchResult,
7
+ type IteratorConfig,
8
+ type OnCompleteArgs,
9
+ type ProcessBatchArgs,
10
+ } from "./index";
11
+
12
+ describe("BatchProcessor client", () => {
13
+ test("exports BatchProcessor class", () => {
14
+ expect(BatchProcessor).toBeDefined();
15
+ expect(typeof BatchProcessor).toBe("function");
16
+ });
17
+
18
+ test("BatchProcessor constructor accepts component API without config", () => {
19
+ // Create a mock component API
20
+ const mockComponent = {
21
+ lib: {
22
+ addItems: {} as any,
23
+ flushBatch: {} as any,
24
+ getBatchStatus: {} as any,
25
+ getFlushHistory: {} as any,
26
+ deleteBatch: {} as any,
27
+ startIteratorJob: {} as any,
28
+ pauseIteratorJob: {} as any,
29
+ resumeIteratorJob: {} as any,
30
+ cancelIteratorJob: {} as any,
31
+ getIteratorJobStatus: {} as any,
32
+ listIteratorJobs: {} as any,
33
+ deleteIteratorJob: {} as any,
34
+ },
35
+ };
36
+
37
+ const processor = new BatchProcessor(mockComponent);
38
+ expect(processor).toBeInstanceOf(BatchProcessor);
39
+ });
40
+
41
+ test("BatchProcessor constructor accepts component API with config", () => {
42
+ const mockComponent = {
43
+ lib: {
44
+ addItems: {} as any,
45
+ flushBatch: {} as any,
46
+ getBatchStatus: {} as any,
47
+ getFlushHistory: {} as any,
48
+ deleteBatch: {} as any,
49
+ startIteratorJob: {} as any,
50
+ pauseIteratorJob: {} as any,
51
+ resumeIteratorJob: {} as any,
52
+ cancelIteratorJob: {} as any,
53
+ getIteratorJobStatus: {} as any,
54
+ listIteratorJobs: {} as any,
55
+ deleteIteratorJob: {} as any,
56
+ },
57
+ };
58
+
59
+ const processor = new BatchProcessor(mockComponent, {
60
+ maxBatchSize: 100,
61
+ flushIntervalMs: 30000,
62
+ processBatch: {} as any, // Mock FunctionReference
63
+ });
64
+ expect(processor).toBeInstanceOf(BatchProcessor);
65
+ });
66
+ });
67
+
68
+ describe("type exports", () => {
69
+ test("BatchConfig type is usable", () => {
70
+ const config: BatchConfig = {
71
+ maxBatchSize: 100,
72
+ flushIntervalMs: 30000,
73
+ processBatch: {} as any, // FunctionReference is opaque, use mock
74
+ };
75
+ expect(config.maxBatchSize).toBe(100);
76
+ });
77
+
78
+ test("IteratorConfig type is usable", () => {
79
+ const config: IteratorConfig = {
80
+ batchSize: 50,
81
+ delayBetweenBatchesMs: 100,
82
+ getNextBatch: {} as any, // FunctionReference mock
83
+ processBatch: {} as any, // FunctionReference mock
84
+ onComplete: {} as any, // FunctionReference mock
85
+ maxRetries: 3,
86
+ };
87
+ expect(config.batchSize).toBe(50);
88
+ });
89
+
90
+ test("GetNextBatchResult type is usable", () => {
91
+ const result: GetNextBatchResult<{ id: number }> = {
92
+ items: [{ id: 1 }, { id: 2 }],
93
+ cursor: "cursor-123",
94
+ done: false,
95
+ };
96
+ expect(result.items.length).toBe(2);
97
+ });
98
+
99
+ test("GetNextBatchArgs type is usable", () => {
100
+ const args: GetNextBatchArgs = {
101
+ cursor: "cursor-123",
102
+ batchSize: 100,
103
+ };
104
+ expect(args.batchSize).toBe(100);
105
+ });
106
+
107
+ test("ProcessBatchArgs type is usable", () => {
108
+ const args: ProcessBatchArgs<{ id: number }> = {
109
+ items: [{ id: 1 }],
110
+ };
111
+ expect(args.items.length).toBe(1);
112
+ });
113
+
114
+ test("OnCompleteArgs type is usable", () => {
115
+ const args: OnCompleteArgs = {
116
+ jobId: "job-123",
117
+ processedCount: 1000,
118
+ };
119
+ expect(args.processedCount).toBe(1000);
120
+ });
121
+ });
@@ -0,0 +1,308 @@
1
+ import {
2
+ type FunctionReference,
3
+ type GenericMutationCtx,
4
+ type GenericQueryCtx,
5
+ createFunctionHandle,
6
+ } from "convex/server";
7
+
8
+ export type BatchStatus = "accumulating" | "flushing" | "completed";
9
+ export type JobStatus = "pending" | "running" | "paused" | "completed" | "failed";
10
+
11
+ // User-facing config interfaces (accept FunctionReference)
12
+ export interface BatchConfig<T = unknown> {
13
+ maxBatchSize: number;
14
+ flushIntervalMs: number;
15
+ processBatch: FunctionReference<"action", "internal", { items: T[] }>;
16
+ }
17
+
18
+ export interface IteratorConfig<T = unknown> {
19
+ batchSize: number;
20
+ delayBetweenBatchesMs?: number;
21
+ getNextBatch: FunctionReference<
22
+ "query",
23
+ "internal",
24
+ { cursor: string | undefined; batchSize: number }
25
+ >;
26
+ processBatch: FunctionReference<"action", "internal", { items: T[] }>;
27
+ onComplete?: FunctionReference<"mutation", "internal", { jobId: string; processedCount: number }>;
28
+ maxRetries?: number;
29
+ }
30
+
31
+ // Internal config interfaces (use string handles for component API)
32
+ interface InternalBatchConfig {
33
+ maxBatchSize: number;
34
+ flushIntervalMs: number;
35
+ processBatchHandle: string;
36
+ }
37
+
38
+ interface InternalIteratorConfig {
39
+ batchSize: number;
40
+ delayBetweenBatchesMs?: number;
41
+ getNextBatchHandle: string;
42
+ processBatchHandle: string;
43
+ onCompleteHandle?: string;
44
+ maxRetries?: number;
45
+ }
46
+
47
+ export interface BatchResult {
48
+ batchId: string;
49
+ itemCount: number;
50
+ flushed: boolean;
51
+ status: BatchStatus;
52
+ }
53
+
54
+ export interface FlushResult {
55
+ batchId: string;
56
+ itemCount: number;
57
+ flushed: boolean;
58
+ status?: BatchStatus;
59
+ reason?: string;
60
+ }
61
+
62
+ export interface BatchStatusResult {
63
+ batchId: string;
64
+ itemCount: number;
65
+ status: BatchStatus;
66
+ createdAt: number;
67
+ lastUpdatedAt: number;
68
+ config: BatchConfig;
69
+ }
70
+
71
+ export interface JobResult {
72
+ jobId: string;
73
+ status: JobStatus;
74
+ }
75
+
76
+ export interface JobStatusResult {
77
+ jobId: string;
78
+ status: JobStatus;
79
+ processedCount: number;
80
+ cursor?: string;
81
+ retryCount: number;
82
+ errorMessage?: string;
83
+ createdAt: number;
84
+ lastRunAt?: number;
85
+ config: {
86
+ batchSize: number;
87
+ delayBetweenBatchesMs: number;
88
+ };
89
+ }
90
+
91
+ export interface JobListItem {
92
+ jobId: string;
93
+ status: JobStatus;
94
+ processedCount: number;
95
+ createdAt: number;
96
+ lastRunAt?: number;
97
+ errorMessage?: string;
98
+ }
99
+
100
+ export interface FlushHistoryItem {
101
+ batchId: string;
102
+ itemCount: number;
103
+ flushedAt: number;
104
+ durationMs: number;
105
+ success: boolean;
106
+ errorMessage?: string;
107
+ }
108
+
109
+ export interface BatchProcessorAPI {
110
+ lib: {
111
+ addItems: FunctionReference<
112
+ "mutation",
113
+ "internal",
114
+ { batchId: string; items: unknown[]; config: InternalBatchConfig },
115
+ BatchResult
116
+ >;
117
+ flushBatch: FunctionReference<"mutation", "internal", { batchId: string }, FlushResult>;
118
+ getBatchStatus: FunctionReference<
119
+ "query",
120
+ "internal",
121
+ { batchId: string },
122
+ BatchStatusResult | null
123
+ >;
124
+ getFlushHistory: FunctionReference<
125
+ "query",
126
+ "internal",
127
+ { batchId: string; limit?: number },
128
+ FlushHistoryItem[]
129
+ >;
130
+ deleteBatch: FunctionReference<
131
+ "mutation",
132
+ "internal",
133
+ { batchId: string },
134
+ { deleted: boolean; reason?: string }
135
+ >;
136
+ startIteratorJob: FunctionReference<
137
+ "mutation",
138
+ "internal",
139
+ { jobId: string; config: InternalIteratorConfig },
140
+ JobResult
141
+ >;
142
+ pauseIteratorJob: FunctionReference<"mutation", "internal", { jobId: string }, JobResult>;
143
+ resumeIteratorJob: FunctionReference<"mutation", "internal", { jobId: string }, JobResult>;
144
+ cancelIteratorJob: FunctionReference<
145
+ "mutation",
146
+ "internal",
147
+ { jobId: string },
148
+ JobResult & { reason?: string }
149
+ >;
150
+ getIteratorJobStatus: FunctionReference<
151
+ "query",
152
+ "internal",
153
+ { jobId: string },
154
+ JobStatusResult | null
155
+ >;
156
+ listIteratorJobs: FunctionReference<
157
+ "query",
158
+ "internal",
159
+ { status?: JobStatus; limit?: number },
160
+ JobListItem[]
161
+ >;
162
+ deleteIteratorJob: FunctionReference<
163
+ "mutation",
164
+ "internal",
165
+ { jobId: string },
166
+ { deleted: boolean; reason?: string }
167
+ >;
168
+ };
169
+ }
170
+
171
+ export class BatchProcessor<T = unknown> {
172
+ private component: BatchProcessorAPI;
173
+ private config?: BatchConfig<T>;
174
+ private processBatchHandle: string | null = null;
175
+
176
+ constructor(component: BatchProcessorAPI, config?: BatchConfig<T>) {
177
+ this.component = component;
178
+ this.config = config;
179
+ }
180
+
181
+ async addItems(ctx: GenericMutationCtx<any>, batchId: string, items: T[]): Promise<BatchResult> {
182
+ if (!this.config) {
183
+ throw new Error(
184
+ "BatchProcessor config with processBatch is required to use addItems. Pass config to the constructor.",
185
+ );
186
+ }
187
+
188
+ if (!this.processBatchHandle) {
189
+ this.processBatchHandle = await createFunctionHandle(this.config.processBatch);
190
+ }
191
+
192
+ const internalConfig: InternalBatchConfig = {
193
+ maxBatchSize: this.config.maxBatchSize,
194
+ flushIntervalMs: this.config.flushIntervalMs,
195
+ processBatchHandle: this.processBatchHandle,
196
+ };
197
+
198
+ return await ctx.runMutation(this.component.lib.addItems, {
199
+ batchId,
200
+ items,
201
+ config: internalConfig,
202
+ });
203
+ }
204
+
205
+ async flush(ctx: GenericMutationCtx<any>, batchId: string): Promise<FlushResult> {
206
+ return await ctx.runMutation(this.component.lib.flushBatch, { batchId });
207
+ }
208
+
209
+ async getBatchStatus(
210
+ ctx: GenericQueryCtx<any>,
211
+ batchId: string,
212
+ ): Promise<BatchStatusResult | null> {
213
+ return await ctx.runQuery(this.component.lib.getBatchStatus, { batchId });
214
+ }
215
+
216
+ async getFlushHistory(
217
+ ctx: GenericQueryCtx<any>,
218
+ batchId: string,
219
+ limit?: number,
220
+ ): Promise<FlushHistoryItem[]> {
221
+ return await ctx.runQuery(this.component.lib.getFlushHistory, { batchId, limit });
222
+ }
223
+
224
+ async deleteBatch(
225
+ ctx: GenericMutationCtx<any>,
226
+ batchId: string,
227
+ ): Promise<{ deleted: boolean; reason?: string }> {
228
+ return await ctx.runMutation(this.component.lib.deleteBatch, { batchId });
229
+ }
230
+
231
+ async startIterator<T>(
232
+ ctx: GenericMutationCtx<any>,
233
+ jobId: string,
234
+ config: IteratorConfig<T>,
235
+ ): Promise<JobResult> {
236
+ const internalConfig: InternalIteratorConfig = {
237
+ batchSize: config.batchSize,
238
+ delayBetweenBatchesMs: config.delayBetweenBatchesMs,
239
+ getNextBatchHandle: await createFunctionHandle(config.getNextBatch),
240
+ processBatchHandle: await createFunctionHandle(config.processBatch),
241
+ onCompleteHandle: config.onComplete
242
+ ? await createFunctionHandle(config.onComplete)
243
+ : undefined,
244
+ maxRetries: config.maxRetries,
245
+ };
246
+
247
+ return await ctx.runMutation(this.component.lib.startIteratorJob, {
248
+ jobId,
249
+ config: internalConfig,
250
+ });
251
+ }
252
+
253
+ async pauseIterator(ctx: GenericMutationCtx<any>, jobId: string): Promise<JobResult> {
254
+ return await ctx.runMutation(this.component.lib.pauseIteratorJob, { jobId });
255
+ }
256
+
257
+ async resumeIterator(ctx: GenericMutationCtx<any>, jobId: string): Promise<JobResult> {
258
+ return await ctx.runMutation(this.component.lib.resumeIteratorJob, { jobId });
259
+ }
260
+
261
+ async cancelIterator(
262
+ ctx: GenericMutationCtx<any>,
263
+ jobId: string,
264
+ ): Promise<JobResult & { reason?: string }> {
265
+ return await ctx.runMutation(this.component.lib.cancelIteratorJob, { jobId });
266
+ }
267
+
268
+ async getIteratorStatus(
269
+ ctx: GenericQueryCtx<any>,
270
+ jobId: string,
271
+ ): Promise<JobStatusResult | null> {
272
+ return await ctx.runQuery(this.component.lib.getIteratorJobStatus, { jobId });
273
+ }
274
+
275
+ async listIteratorJobs(
276
+ ctx: GenericQueryCtx<any>,
277
+ options?: { status?: JobStatus; limit?: number },
278
+ ): Promise<JobListItem[]> {
279
+ return await ctx.runQuery(this.component.lib.listIteratorJobs, options ?? {});
280
+ }
281
+
282
+ async deleteIteratorJob(
283
+ ctx: GenericMutationCtx<any>,
284
+ jobId: string,
285
+ ): Promise<{ deleted: boolean; reason?: string }> {
286
+ return await ctx.runMutation(this.component.lib.deleteIteratorJob, { jobId });
287
+ }
288
+ }
289
+
290
+ export interface GetNextBatchResult<T = unknown> {
291
+ items: T[];
292
+ cursor: string | undefined;
293
+ done: boolean;
294
+ }
295
+
296
+ export interface GetNextBatchArgs {
297
+ cursor: string | undefined;
298
+ batchSize: number;
299
+ }
300
+
301
+ export interface ProcessBatchArgs<T = unknown> {
302
+ items: T[];
303
+ }
304
+
305
+ export interface OnCompleteArgs {
306
+ jobId: string;
307
+ processedCount: number;
308
+ }