@atproto/sync 0.3.2 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/runner/memory-runner.d.ts +7 -8
- package/dist/runner/memory-runner.d.ts.map +1 -1
- package/dist/runner/memory-runner.js +12 -5
- package/dist/runner/memory-runner.js.map +1 -1
- package/package.json +4 -4
- package/src/runner/memory-runner.ts +16 -9
- package/tests/firehose.test.ts +1 -1
- package/tests/runner.test.ts +10 -10
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
# @atproto/sync
|
|
2
2
|
|
|
3
|
+
## 0.3.3
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- [#5117](https://github.com/bluesky-social/atproto/pull/5117) [`8a4c88b`](https://github.com/bluesky-social/atproto/commit/8a4c88b0f63fb2d82b1391d64c54e0c760fac48b) Thanks [@matthieusieben](https://github.com/matthieusieben)! - Improve concurrency management in MemoryRunner
|
|
8
|
+
|
|
3
9
|
## 0.3.2
|
|
4
10
|
|
|
5
11
|
### Patch Changes
|
|
@@ -1,25 +1,24 @@
|
|
|
1
|
-
import PQueue from 'p-queue';
|
|
2
|
-
import { ConsecutiveList } from './consecutive-list.js';
|
|
3
1
|
import { EventRunner } from './types.js';
|
|
4
2
|
export type MemoryRunnerOptions = {
|
|
5
3
|
setCursor?: (cursor: number) => Promise<void>;
|
|
6
4
|
concurrency?: number;
|
|
7
5
|
startCursor?: number;
|
|
8
6
|
};
|
|
9
|
-
type Queue = InstanceType<typeof PQueue>;
|
|
10
7
|
export declare class MemoryRunner implements EventRunner {
|
|
11
8
|
opts: MemoryRunnerOptions;
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
9
|
+
private destroyed;
|
|
10
|
+
private readonly consecutive;
|
|
11
|
+
private readonly mainQueue;
|
|
12
|
+
private readonly partitions;
|
|
13
|
+
private cursor;
|
|
16
14
|
constructor(opts?: MemoryRunnerOptions);
|
|
17
15
|
getCursor(): number | undefined;
|
|
16
|
+
/** @deprecated internal use only */
|
|
17
|
+
get partitionCount(): number;
|
|
18
18
|
addTask(partitionId: string, task: () => Promise<void>): Promise<void>;
|
|
19
19
|
private getPartition;
|
|
20
20
|
trackEvent(did: string, seq: number, handler: () => Promise<void>): Promise<void>;
|
|
21
21
|
processAll(): Promise<void>;
|
|
22
22
|
destroy(): Promise<void>;
|
|
23
23
|
}
|
|
24
|
-
export {};
|
|
25
24
|
//# sourceMappingURL=memory-runner.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory-runner.d.ts","sourceRoot":"","sources":["../../src/runner/memory-runner.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"memory-runner.d.ts","sourceRoot":"","sources":["../../src/runner/memory-runner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAExC,MAAM,MAAM,mBAAmB,GAAG;IAChC,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IAC7C,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB,CAAA;AAMD,qBAAa,YAAa,YAAW,WAAW;IAO3B,IAAI,EAAE,mBAAmB;IAN5C,OAAO,CAAC,SAAS,CAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAgC;IAC5D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAO;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAgC;IAC3D,OAAO,CAAC,MAAM,CAAoB;IAElC,YAAmB,IAAI,GAAE,mBAAwB,EAGhD;IAED,SAAS,uBAER;IAED,oCAAoC;IACpC,IAAI,cAAc,WAEjB;IAEK,OAAO,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBAK3D;IAED,OAAO,CAAC,YAAY;IAUd,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBAatE;IAEK,UAAU,kBAGf;IAEK,OAAO,kBAKZ;CACF"}
|
|
@@ -5,6 +5,7 @@ import { ConsecutiveList } from './consecutive-list.js';
|
|
|
5
5
|
export class MemoryRunner {
|
|
6
6
|
constructor(opts = {}) {
|
|
7
7
|
this.opts = opts;
|
|
8
|
+
this.destroyed = false;
|
|
8
9
|
this.consecutive = new ConsecutiveList();
|
|
9
10
|
this.partitions = new Map();
|
|
10
11
|
this.mainQueue = new PQueue({ concurrency: opts.concurrency ?? Infinity });
|
|
@@ -13,8 +14,12 @@ export class MemoryRunner {
|
|
|
13
14
|
getCursor() {
|
|
14
15
|
return this.cursor;
|
|
15
16
|
}
|
|
17
|
+
/** @deprecated internal use only */
|
|
18
|
+
get partitionCount() {
|
|
19
|
+
return this.partitions.size;
|
|
20
|
+
}
|
|
16
21
|
async addTask(partitionId, task) {
|
|
17
|
-
if (this.
|
|
22
|
+
if (this.destroyed)
|
|
18
23
|
return;
|
|
19
24
|
return this.mainQueue.add(() => {
|
|
20
25
|
return this.getPartition(partitionId).add(task);
|
|
@@ -30,7 +35,7 @@ export class MemoryRunner {
|
|
|
30
35
|
return partition;
|
|
31
36
|
}
|
|
32
37
|
async trackEvent(did, seq, handler) {
|
|
33
|
-
if (this.
|
|
38
|
+
if (this.destroyed)
|
|
34
39
|
return;
|
|
35
40
|
const item = this.consecutive.push(seq);
|
|
36
41
|
await this.addTask(did, async () => {
|
|
@@ -45,13 +50,15 @@ export class MemoryRunner {
|
|
|
45
50
|
});
|
|
46
51
|
}
|
|
47
52
|
async processAll() {
|
|
48
|
-
|
|
53
|
+
const queue = this.mainQueue;
|
|
54
|
+
while (queue.size || queue.pending)
|
|
55
|
+
await queue.onIdle();
|
|
49
56
|
}
|
|
50
57
|
async destroy() {
|
|
51
|
-
this.
|
|
58
|
+
this.destroyed = true;
|
|
52
59
|
this.mainQueue.clear();
|
|
53
60
|
this.partitions.forEach((p) => p.clear());
|
|
54
|
-
await this.
|
|
61
|
+
await this.processAll();
|
|
55
62
|
}
|
|
56
63
|
}
|
|
57
64
|
//# sourceMappingURL=memory-runner.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory-runner.js","sourceRoot":"","sources":["../../src/runner/memory-runner.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,SAAS,CAAA;AAC5B,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AAWvD,+EAA+E;AAC/E,2EAA2E;AAC3E,MAAM,OAAO,YAAY;
|
|
1
|
+
{"version":3,"file":"memory-runner.js","sourceRoot":"","sources":["../../src/runner/memory-runner.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,SAAS,CAAA;AAC5B,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AAWvD,+EAA+E;AAC/E,2EAA2E;AAC3E,MAAM,OAAO,YAAY;IAOvB,YAAmB,IAAI,GAAwB,EAAE;QAA9B,SAAI,GAAJ,IAAI,CAA0B;QANzC,cAAS,GAAG,KAAK,CAAA;QACR,gBAAW,GAAG,IAAI,eAAe,EAAU,CAAA;QAE3C,eAAU,GAAuB,IAAI,GAAG,EAAE,CAAA;QAIzD,IAAI,CAAC,SAAS,GAAG,IAAI,MAAM,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,QAAQ,EAAE,CAAC,CAAA;QAC1E,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,CAAA;IAChC,CAAC;IAED,SAAS;QACP,OAAO,IAAI,CAAC,MAAM,CAAA;IACpB,CAAC;IAED,oCAAoC;IACpC,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAA;IAC7B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,WAAmB,EAAE,IAAyB;QAC1D,IAAI,IAAI,CAAC,SAAS;YAAE,OAAM;QAC1B,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;YAC7B,OAAO,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACjD,CAAC,CAAC,CAAA;IACJ,CAAC;IAEO,YAAY,CAAC,WAAmB;QACtC,IAAI,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;QAChD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,SAAS,GAAG,IAAI,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAA;YAC1C,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAA;YACjE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;QAC7C,CAAC;QACD,OAAO,SAAS,CAAA;IAClB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,GAAW,EAAE,GAAW,EAAE,OAA4B;QACrE,IAAI,IAAI,CAAC,SAAS;YAAE,OAAM;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACvC,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,IAAI,EAAE;YACjC,MAAM,OAAO,EAAE,CAAA;YACf,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;YACrC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;gBACpB,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;oBACxB,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBACxC,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,KAAK,CAAC,UAAU;QACd,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAA;QAC5B,OAAO,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,OAAO;YAAE,MAAM,KAAK,CAAC,MAAM,EAAE,CAAA;IAC1D,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;QACrB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAA;QACtB,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAA;QACzC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAA;IACzB,CAAC;CACF","sourcesContent":["import PQueue from 'p-queue'\nimport { ConsecutiveList } from './consecutive-list.js'\nimport { EventRunner } from './types.js'\n\nexport type MemoryRunnerOptions = {\n setCursor?: (cursor: number) => Promise<void>\n concurrency?: number\n startCursor?: number\n}\n\ntype Queue = InstanceType<typeof PQueue>\n\n// A queue with arbitrarily many partitions, each processing work sequentially.\n// Partitions are created lazily and taken out of memory when they go idle.\nexport class MemoryRunner implements EventRunner {\n private destroyed = false\n private readonly consecutive = new ConsecutiveList<number>()\n private readonly mainQueue: Queue\n private readonly partitions: Map<string, Queue> = new Map()\n private cursor: number | undefined\n\n constructor(public opts: MemoryRunnerOptions = {}) {\n this.mainQueue = new PQueue({ concurrency: opts.concurrency ?? Infinity })\n this.cursor = opts.startCursor\n }\n\n getCursor() {\n return this.cursor\n }\n\n /** @deprecated internal use only */\n get partitionCount() {\n return this.partitions.size\n }\n\n async addTask(partitionId: string, task: () => Promise<void>) {\n if (this.destroyed) return\n return this.mainQueue.add(() => {\n return this.getPartition(partitionId).add(task)\n })\n }\n\n private getPartition(partitionId: string) {\n let partition = this.partitions.get(partitionId)\n if (!partition) {\n partition = new PQueue({ concurrency: 1 })\n partition.once('idle', () => this.partitions.delete(partitionId))\n this.partitions.set(partitionId, partition)\n }\n return partition\n }\n\n async trackEvent(did: string, seq: number, handler: () => Promise<void>) {\n if (this.destroyed) return\n const item = this.consecutive.push(seq)\n await this.addTask(did, async () => {\n await handler()\n const latest = item.complete().at(-1)\n if (latest !== undefined) {\n this.cursor = latest\n if (this.opts.setCursor) {\n await this.opts.setCursor(this.cursor)\n }\n }\n })\n }\n\n async processAll() {\n const queue = this.mainQueue\n while (queue.size || queue.pending) await queue.onIdle()\n }\n\n async destroy() {\n this.destroyed = true\n this.mainQueue.clear()\n this.partitions.forEach((p) => p.clear())\n await this.processAll()\n }\n}\n"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@atproto/sync",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.3",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"description": "atproto sync library",
|
|
6
6
|
"keywords": [
|
|
@@ -21,12 +21,12 @@
|
|
|
21
21
|
"dependencies": {
|
|
22
22
|
"p-queue": "^8.0.0",
|
|
23
23
|
"ws": "^8.12.0",
|
|
24
|
-
"@atproto/common": "^0.6.3",
|
|
25
24
|
"@atproto/identity": "^0.5.1",
|
|
25
|
+
"@atproto/common": "^0.6.3",
|
|
26
26
|
"@atproto/lex": "^0.1.4",
|
|
27
|
+
"@atproto/repo": "^0.10.1",
|
|
27
28
|
"@atproto/syntax": "^0.6.2",
|
|
28
|
-
"@atproto/xrpc-server": "^0.11.2"
|
|
29
|
-
"@atproto/repo": "^0.10.1"
|
|
29
|
+
"@atproto/xrpc-server": "^0.11.2"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
32
32
|
"@types/ws": "^8.5.4",
|
|
@@ -13,10 +13,11 @@ type Queue = InstanceType<typeof PQueue>
|
|
|
13
13
|
// A queue with arbitrarily many partitions, each processing work sequentially.
|
|
14
14
|
// Partitions are created lazily and taken out of memory when they go idle.
|
|
15
15
|
export class MemoryRunner implements EventRunner {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
16
|
+
private destroyed = false
|
|
17
|
+
private readonly consecutive = new ConsecutiveList<number>()
|
|
18
|
+
private readonly mainQueue: Queue
|
|
19
|
+
private readonly partitions: Map<string, Queue> = new Map()
|
|
20
|
+
private cursor: number | undefined
|
|
20
21
|
|
|
21
22
|
constructor(public opts: MemoryRunnerOptions = {}) {
|
|
22
23
|
this.mainQueue = new PQueue({ concurrency: opts.concurrency ?? Infinity })
|
|
@@ -27,8 +28,13 @@ export class MemoryRunner implements EventRunner {
|
|
|
27
28
|
return this.cursor
|
|
28
29
|
}
|
|
29
30
|
|
|
31
|
+
/** @deprecated internal use only */
|
|
32
|
+
get partitionCount() {
|
|
33
|
+
return this.partitions.size
|
|
34
|
+
}
|
|
35
|
+
|
|
30
36
|
async addTask(partitionId: string, task: () => Promise<void>) {
|
|
31
|
-
if (this.
|
|
37
|
+
if (this.destroyed) return
|
|
32
38
|
return this.mainQueue.add(() => {
|
|
33
39
|
return this.getPartition(partitionId).add(task)
|
|
34
40
|
})
|
|
@@ -45,7 +51,7 @@ export class MemoryRunner implements EventRunner {
|
|
|
45
51
|
}
|
|
46
52
|
|
|
47
53
|
async trackEvent(did: string, seq: number, handler: () => Promise<void>) {
|
|
48
|
-
if (this.
|
|
54
|
+
if (this.destroyed) return
|
|
49
55
|
const item = this.consecutive.push(seq)
|
|
50
56
|
await this.addTask(did, async () => {
|
|
51
57
|
await handler()
|
|
@@ -60,13 +66,14 @@ export class MemoryRunner implements EventRunner {
|
|
|
60
66
|
}
|
|
61
67
|
|
|
62
68
|
async processAll() {
|
|
63
|
-
|
|
69
|
+
const queue = this.mainQueue
|
|
70
|
+
while (queue.size || queue.pending) await queue.onIdle()
|
|
64
71
|
}
|
|
65
72
|
|
|
66
73
|
async destroy() {
|
|
67
|
-
this.
|
|
74
|
+
this.destroyed = true
|
|
68
75
|
this.mainQueue.clear()
|
|
69
76
|
this.partitions.forEach((p) => p.clear())
|
|
70
|
-
await this.
|
|
77
|
+
await this.processAll()
|
|
71
78
|
}
|
|
72
79
|
}
|
package/tests/firehose.test.ts
CHANGED
package/tests/runner.test.ts
CHANGED
|
@@ -48,7 +48,7 @@ describe('EventRunner utils', () => {
|
|
|
48
48
|
runner.addTask('1', async () => {
|
|
49
49
|
complete[1].push(3)
|
|
50
50
|
})
|
|
51
|
-
expect(runner.
|
|
51
|
+
expect(runner.partitionCount).toEqual(1)
|
|
52
52
|
// partition 2 items complete quickly except the last, which is slowest of all events.
|
|
53
53
|
runner.addTask('2', async () => {
|
|
54
54
|
await wait(1)
|
|
@@ -65,13 +65,13 @@ describe('EventRunner utils', () => {
|
|
|
65
65
|
await wait(60)
|
|
66
66
|
complete[2].push(4)
|
|
67
67
|
})
|
|
68
|
-
expect(runner.
|
|
69
|
-
await runner.
|
|
68
|
+
expect(runner.partitionCount).toEqual(2)
|
|
69
|
+
await runner.processAll()
|
|
70
70
|
expect(complete).toEqual({
|
|
71
71
|
1: [1, 2, 3],
|
|
72
72
|
2: [1, 2, 3, 4],
|
|
73
73
|
})
|
|
74
|
-
expect(runner.
|
|
74
|
+
expect(runner.partitionCount).toEqual(0)
|
|
75
75
|
})
|
|
76
76
|
|
|
77
77
|
it('limits overall concurrency.', async () => {
|
|
@@ -96,10 +96,10 @@ describe('EventRunner utils', () => {
|
|
|
96
96
|
complete.push(22)
|
|
97
97
|
})
|
|
98
98
|
// only partition 1 exists so far due to the concurrency
|
|
99
|
-
expect(runner.
|
|
100
|
-
await runner.
|
|
99
|
+
expect(runner.partitionCount).toEqual(1)
|
|
100
|
+
await runner.processAll()
|
|
101
101
|
expect(complete).toEqual([11, 21, 12, 22])
|
|
102
|
-
expect(runner.
|
|
102
|
+
expect(runner.partitionCount).toEqual(0)
|
|
103
103
|
})
|
|
104
104
|
|
|
105
105
|
it('settles with many items.', async () => {
|
|
@@ -114,8 +114,8 @@ describe('EventRunner utils', () => {
|
|
|
114
114
|
complete.push({ partition, id: i })
|
|
115
115
|
})
|
|
116
116
|
}
|
|
117
|
-
expect(runner.
|
|
118
|
-
await runner.
|
|
117
|
+
expect(runner.partitionCount).toBeLessThanOrEqual(partitions.size)
|
|
118
|
+
await runner.processAll()
|
|
119
119
|
expect(complete.length).toEqual(500)
|
|
120
120
|
for (const partition of partitions) {
|
|
121
121
|
const ids = complete
|
|
@@ -123,7 +123,7 @@ describe('EventRunner utils', () => {
|
|
|
123
123
|
.map((item) => item.id)
|
|
124
124
|
expect(ids).toEqual([...ids].sort((a, b) => a - b))
|
|
125
125
|
}
|
|
126
|
-
expect(runner.
|
|
126
|
+
expect(runner.partitionCount).toEqual(0)
|
|
127
127
|
})
|
|
128
128
|
})
|
|
129
129
|
})
|