@opendatalabs/personal-server-ts-core 1.26.0 → 1.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/index.d.ts.map +1 -1
- package/dist/api/index.js +62 -1
- package/dist/api/index.js.map +1 -1
- package/dist/contracts/data.d.ts +5 -0
- package/dist/contracts/data.d.ts.map +1 -1
- package/dist/contracts/data.js +3 -1
- package/dist/contracts/data.js.map +1 -1
- package/dist/derivatives/scheduler.d.ts.map +1 -1
- package/dist/derivatives/scheduler.js +18 -1
- package/dist/derivatives/scheduler.js.map +1 -1
- package/dist/ports/index.d.ts +6 -0
- package/dist/ports/index.d.ts.map +1 -1
- package/dist/ports/index.js.map +1 -1
- package/dist/storage/blocks/cursor.d.ts +5 -1
- package/dist/storage/blocks/cursor.d.ts.map +1 -1
- package/dist/storage/blocks/cursor.js +13 -1
- package/dist/storage/blocks/cursor.js.map +1 -1
- package/dist/storage/blocks/page.d.ts +18 -0
- package/dist/storage/blocks/page.d.ts.map +1 -0
- package/dist/storage/blocks/page.js +113 -0
- package/dist/storage/blocks/page.js.map +1 -0
- package/dist/storage/legacy-projection.d.ts +85 -0
- package/dist/storage/legacy-projection.d.ts.map +1 -0
- package/dist/storage/legacy-projection.js +552 -0
- package/dist/storage/legacy-projection.js.map +1 -0
- package/package.json +5 -1
package/dist/ports/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/ports/index.ts"],"names":[],"mappings":";
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/ports/index.ts"],"names":[],"mappings":";AAsUA,4EAA4E;AAC5E,wEAAwE;AACxE,qEAAqE;AACrE,2EAA2E;AAC3E,4EAA4E"}
|
|
@@ -4,14 +4,17 @@ export interface DataBlockCursor {
|
|
|
4
4
|
collectedAt: string;
|
|
5
5
|
blockIndex: number;
|
|
6
6
|
intraBlockOffset?: number;
|
|
7
|
+
/** Identifies a read-time view of the version (see readBuiltScopeBlocks). */
|
|
8
|
+
view?: string;
|
|
7
9
|
}
|
|
8
10
|
export interface EncodeDataBlockCursorInput {
|
|
9
11
|
scope: string;
|
|
10
12
|
collectedAt: string;
|
|
11
13
|
blockIndex: number;
|
|
12
14
|
intraBlockOffset?: number;
|
|
15
|
+
view?: string;
|
|
13
16
|
}
|
|
14
|
-
export type DataBlockCursorErrorCode = "cursor_malformed" | "cursor_unsupported_version" | "cursor_scope_mismatch" | "cursor_collected_at_mismatch";
|
|
17
|
+
export type DataBlockCursorErrorCode = "cursor_malformed" | "cursor_unsupported_version" | "cursor_scope_mismatch" | "cursor_collected_at_mismatch" | "cursor_view_mismatch";
|
|
15
18
|
export interface DataBlockCursorError {
|
|
16
19
|
code: DataBlockCursorErrorCode;
|
|
17
20
|
message: string;
|
|
@@ -29,5 +32,6 @@ export declare function decodeDataBlockCursor(cursor: string): DecodeDataBlockCu
|
|
|
29
32
|
export declare function validateDataBlockCursor(cursor: string, expected: {
|
|
30
33
|
scope: string;
|
|
31
34
|
collectedAt: string;
|
|
35
|
+
view?: string;
|
|
32
36
|
}): ValidateDataBlockCursorResult;
|
|
33
37
|
//# sourceMappingURL=cursor.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cursor.d.ts","sourceRoot":"","sources":["../../../src/storage/blocks/cursor.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,CAAC,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"cursor.d.ts","sourceRoot":"","sources":["../../../src/storage/blocks/cursor.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,CAAC,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,6EAA6E;IAC7E,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,0BAA0B;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,MAAM,wBAAwB,GAChC,kBAAkB,GAClB,4BAA4B,GAC5B,uBAAuB,GACvB,8BAA8B,GAC9B,sBAAsB,CAAC;AAE3B,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,wBAAwB,CAAC;IAC/B,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,MAAM,2BAA2B,GACnC;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,eAAe,CAAA;CAAE,GACrC;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,oBAAoB,CAAA;CAAE,CAAC;AAE/C,MAAM,MAAM,6BAA6B,GAAG,2BAA2B,CAAC;AAIxE,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,0BAA0B,GAChC,MAAM,CAoBR;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,MAAM,GACb,2BAA2B,CA6D7B;AAED,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC9D,6BAA6B,CAoC/B"}
|
|
@@ -14,6 +14,7 @@ export function encodeDataBlockCursor(input) {
|
|
|
14
14
|
...(input.intraBlockOffset === undefined
|
|
15
15
|
? {}
|
|
16
16
|
: { intraBlockOffset: input.intraBlockOffset }),
|
|
17
|
+
...(input.view === undefined ? {} : { view: input.view }),
|
|
17
18
|
}));
|
|
18
19
|
}
|
|
19
20
|
export function decodeDataBlockCursor(cursor) {
|
|
@@ -52,7 +53,8 @@ export function decodeDataBlockCursor(cursor) {
|
|
|
52
53
|
candidate.collectedAt.length === 0 ||
|
|
53
54
|
!isNonNegativeInteger(candidate.blockIndex) ||
|
|
54
55
|
(candidate.intraBlockOffset !== undefined &&
|
|
55
|
-
!isNonNegativeInteger(candidate.intraBlockOffset))
|
|
56
|
+
!isNonNegativeInteger(candidate.intraBlockOffset)) ||
|
|
57
|
+
(candidate.view !== undefined && typeof candidate.view !== "string")) {
|
|
56
58
|
return malformedCursor("Cursor payload has invalid fields");
|
|
57
59
|
}
|
|
58
60
|
return {
|
|
@@ -65,6 +67,7 @@ export function decodeDataBlockCursor(cursor) {
|
|
|
65
67
|
...(candidate.intraBlockOffset === undefined
|
|
66
68
|
? {}
|
|
67
69
|
: { intraBlockOffset: candidate.intraBlockOffset }),
|
|
70
|
+
...(typeof candidate.view === "string" ? { view: candidate.view } : {}),
|
|
68
71
|
},
|
|
69
72
|
};
|
|
70
73
|
}
|
|
@@ -90,6 +93,15 @@ export function validateDataBlockCursor(cursor, expected) {
|
|
|
90
93
|
},
|
|
91
94
|
};
|
|
92
95
|
}
|
|
96
|
+
if (decoded.cursor.view !== expected.view) {
|
|
97
|
+
return {
|
|
98
|
+
ok: false,
|
|
99
|
+
error: {
|
|
100
|
+
code: "cursor_view_mismatch",
|
|
101
|
+
message: "The data behind this cursor changed; restart without a cursor",
|
|
102
|
+
},
|
|
103
|
+
};
|
|
104
|
+
}
|
|
93
105
|
return decoded;
|
|
94
106
|
}
|
|
95
107
|
function encodeBase64Url(value) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cursor.js","sourceRoot":"","sources":["../../../src/storage/blocks/cursor.ts"],"names":[],"mappings":"AAAA,MAAM,yBAAyB,GAAG,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"cursor.js","sourceRoot":"","sources":["../../../src/storage/blocks/cursor.ts"],"names":[],"mappings":"AAAA,MAAM,yBAAyB,GAAG,CAAC,CAAC;AAwCpC,MAAM,UAAU,qBAAqB,CACnC,KAAiC;IAEjC,oBAAoB,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC3C,oBAAoB,CAAC,KAAK,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;IACvD,wBAAwB,CAAC,KAAK,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;IACzD,IAAI,KAAK,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;QACzC,wBAAwB,CAAC,KAAK,CAAC,gBAAgB,EAAE,kBAAkB,CAAC,CAAC;IACvE,CAAC;IAED,OAAO,eAAe,CACpB,IAAI,CAAC,SAAS,CAAC;QACb,CAAC,EAAE,yBAAyB;QAC5B,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,GAAG,CAAC,KAAK,CAAC,gBAAgB,KAAK,SAAS;YACtC,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC;QACjD,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC;KAC1D,CAAC,CACH,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,MAAc;IAEd,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,eAAe,CAAC,oBAAoB,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IACxC,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACrB,OAAO,eAAe,CAAC,+BAA+B,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,eAAe,CAAC,kCAAkC,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACtB,OAAO,eAAe,CAAC,kCAAkC,CAAC,CAAC;IAC7D,CAAC;IAED,MAAM,SAAS,GAAG,MAAmC,CAAC;IACtD,IAAI,SAAS,CAAC,CAAC,KAAK,yBAAyB,EAAE,CAAC;QAC9C,OAAO;YACL,EAAE,EAAE,KAAK;YACT,KAAK,EAAE;gBACL,IAAI,EAAE,4BAA4B;gBAClC,OAAO,EACL,OAAO,SAAS,CAAC,CAAC,KAAK,QAAQ;oBAC7B,CAAC,CAAC,+BAA+B,SAAS,CAAC,CAAC,EAAE;oBAC9C,CAAC,CAAC,sCAAsC;aAC7C;SACF,CAAC;IACJ,CAAC;IAED,IACE,OAAO,SAAS,CAAC,KAAK,KAAK,QAAQ;QACnC,SAAS,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;QAC5B,OAAO,SAAS,CAAC,WAAW,KAAK,QAAQ;QACzC,SAAS,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC;QAClC,CAAC,oBAAoB,CAAC,SAAS,CAAC,UAAU,CAAC;QAC3C,CAAC,SAAS,CAAC,gBAAgB,KAAK,SAAS;YACvC,CAAC,oBAAoB,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;QACpD,CAAC,SAAS,CAAC,IAAI,KAAK,SAAS,IAAI,OAAO,SAAS,CAAC,IAAI,KAAK,QAAQ,CAAC,EACpE,CAAC;QACD,OAAO,eAAe,CAAC,mCAAmC,CAAC,CAAC;IAC9D,CAAC;IAED,OAAO;QACL,EAAE,EAAE,IAAI;QACR,MAAM,EAAE;YACN,OAAO,EAAE,yBAAyB;YAClC,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,WAAW,EAAE,SAAS,CAAC,WAAW;YAClC,UAAU,EAAE,SAAS,CAAC,UAAU;YAChC,GAAG,CAAC,SAAS,CAAC,gBAAgB,KAAK,SAAS;gBAC1C,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,gBAAgB,EAAE,SAAS,CAAC,gBAAgB,EAAE,CAAC;YACrD,GAAG,CAAC,OAAO,SAAS,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACxE;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,MAAc,EACd,QAA+D;IAE/D,MAAM,OAAO,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;IAC9C,IAAI,CAAC,OAAO,CAAC,EAAE;QAAE,OAAO,OAAO,CAAC;IAEhC,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK,EAAE,CAAC;QAC5C,OAAO;YACL,EAAE,EAAE,KAAK;YACT,KAAK,EAAE;gBACL,IAAI,EAAE,uBAAuB;gBAC7B,OAAO,EAAE,6CAA6C;aACvD;SACF,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW,EAAE,CAAC;QACxD,OAAO;YACL,EAAE,EAAE,KAAK;YACT,KAAK,EAAE;gBACL,IAAI,EAAE,8BAA8B;gBACpC,OAAO,EAAE,yDAAyD;aACnE;SACF,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC1C,OAAO;YACL,EAAE,EAAE,KAAK;YACT,KAAK,EAAE;gBACL,IAAI,EAAE,sBAAsB;gBAC5B,OAAO,EACL,+DAA+D;aAClE;SACF,CAAC;IACJ,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,eAAe,CAAC,KAAa;IACpC,MAAM,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC9C,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,MAAM,IAAI,IAAI,KAAK;QAAE,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC9D,OAAO,IAAI,CAAC,MAAM,CAAC;SAChB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AACxB,CAAC;AAED,SAAS,eAAe,CAAC,KAAa;IACpC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC3D,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAC1B,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAC/C,GAAG,CACJ,CAAC;QACF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;QACpE,OAAO,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,OAAe;IACtC,OAAO;QACL,EAAE,EAAE,KAAK;QACT,KAAK,EAAE;YACL,IAAI,EAAE,kBAAkB;YACxB,OAAO;SACR;KACF,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAa,EAAE,KAAa;IACxD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,6BAA6B,CAAC,CAAC;IACzD,CAAC;AACH,CAAC;AAED,SAAS,wBAAwB,CAAC,KAAa,EAAE,KAAa;IAC5D,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,iCAAiC,CAAC,CAAC;IAC7D,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAc;IAC1C,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;AAC5E,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Serve `readScopeBlocks` from blocks already built in memory (for example
|
|
3
|
+
* from an envelope that exists only as a read-time view, with no sidecar on
|
|
4
|
+
* disk). Paging, cursors and block selection behave as in the stored-sidecar
|
|
5
|
+
* implementations.
|
|
6
|
+
*/
|
|
7
|
+
import type { DataBlockManifest, DataScopeBlock, ReadScopeBlocksResponse } from "./types.js";
|
|
8
|
+
export declare function readBuiltScopeBlocks(built: {
|
|
9
|
+
manifest: DataBlockManifest;
|
|
10
|
+
blocks: DataScopeBlock[];
|
|
11
|
+
}, options: {
|
|
12
|
+
cursor?: string;
|
|
13
|
+
maxBytes: number;
|
|
14
|
+
blockIds?: readonly string[];
|
|
15
|
+
/** Bound into every cursor; a cursor from another view is rejected. */
|
|
16
|
+
cursorView?: string;
|
|
17
|
+
}): Promise<ReadScopeBlocksResponse>;
|
|
18
|
+
//# sourceMappingURL=page.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"page.d.ts","sourceRoot":"","sources":["../../../src/storage/blocks/page.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EACV,iBAAiB,EACjB,cAAc,EACd,uBAAuB,EACxB,MAAM,YAAY,CAAC;AAMpB,wBAAsB,oBAAoB,CACxC,KAAK,EAAE;IAAE,QAAQ,EAAE,iBAAiB,CAAC;IAAC,MAAM,EAAE,cAAc,EAAE,CAAA;CAAE,EAChE,OAAO,EAAE;IACP,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC7B,uEAAuE;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,GACA,OAAO,CAAC,uBAAuB,CAAC,CA0FlC"}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Serve `readScopeBlocks` from blocks already built in memory (for example
|
|
3
|
+
* from an envelope that exists only as a read-time view, with no sidecar on
|
|
4
|
+
* disk). Paging, cursors and block selection behave as in the stored-sidecar
|
|
5
|
+
* implementations.
|
|
6
|
+
*/
|
|
7
|
+
import { encodeDataBlockCursor, validateDataBlockCursor } from "./cursor.js";
|
|
8
|
+
import { DataBlockStorageError } from "./errors.js";
|
|
9
|
+
import { selectScopeBlocksByIds } from "./select.js";
|
|
10
|
+
const TEXT_PAGE_MEDIA_TYPE = "text/plain; charset=utf-8";
|
|
11
|
+
const textEncoder = new TextEncoder();
|
|
12
|
+
const textDecoder = new TextDecoder();
|
|
13
|
+
export async function readBuiltScopeBlocks(built, options) {
|
|
14
|
+
const { manifest, blocks: builtBlocks } = built;
|
|
15
|
+
const { scope, collectedAt } = manifest;
|
|
16
|
+
const head = {
|
|
17
|
+
scope,
|
|
18
|
+
collectedAt,
|
|
19
|
+
...(manifest.schemaId ? { schemaId: manifest.schemaId } : {}),
|
|
20
|
+
contentKind: manifest.contentKind,
|
|
21
|
+
};
|
|
22
|
+
const byId = new Map(builtBlocks.map((block) => [block.id, block]));
|
|
23
|
+
if (options.blockIds?.length) {
|
|
24
|
+
const selection = await selectScopeBlocksByIds(manifest, options.blockIds, { maxBytes: options.maxBytes }, async (blockId) => byId.get(blockId) ?? null);
|
|
25
|
+
return {
|
|
26
|
+
...head,
|
|
27
|
+
blocks: selection.blocks,
|
|
28
|
+
warnings: [...manifest.warnings, ...selection.warnings],
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
const cursorResult = options.cursor
|
|
32
|
+
? validateDataBlockCursor(options.cursor, {
|
|
33
|
+
scope,
|
|
34
|
+
collectedAt,
|
|
35
|
+
view: options.cursorView,
|
|
36
|
+
})
|
|
37
|
+
: { ok: true, cursor: null };
|
|
38
|
+
if (!cursorResult.ok) {
|
|
39
|
+
throw new DataBlockStorageError("cursor_invalid", cursorResult.error.message);
|
|
40
|
+
}
|
|
41
|
+
const maxBytes = Math.max(1, options.maxBytes);
|
|
42
|
+
const startIndex = cursorResult.cursor?.blockIndex ?? 0;
|
|
43
|
+
const startOffset = cursorResult.cursor?.intraBlockOffset ?? 0;
|
|
44
|
+
const blocks = [];
|
|
45
|
+
let bytes = 0;
|
|
46
|
+
let nextIndex = startIndex;
|
|
47
|
+
let nextOffset;
|
|
48
|
+
while (nextIndex < manifest.blocks.length) {
|
|
49
|
+
const ref = manifest.blocks[nextIndex];
|
|
50
|
+
const block = ref ? byId.get(ref.id) : undefined;
|
|
51
|
+
if (!ref || !block) {
|
|
52
|
+
nextIndex += 1;
|
|
53
|
+
continue;
|
|
54
|
+
}
|
|
55
|
+
const offset = nextIndex === startIndex ? startOffset : 0;
|
|
56
|
+
if (offset >= ref.sizeBytes) {
|
|
57
|
+
nextIndex += 1;
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
if (blocks.length > 0 && offset === 0 && bytes + ref.sizeBytes > maxBytes) {
|
|
61
|
+
break;
|
|
62
|
+
}
|
|
63
|
+
const page = pageBlock(block, offset, maxBytes - bytes);
|
|
64
|
+
blocks.push(page.block);
|
|
65
|
+
bytes += page.block.sizeBytes;
|
|
66
|
+
if (page.nextOffset !== undefined) {
|
|
67
|
+
nextOffset = page.nextOffset;
|
|
68
|
+
break;
|
|
69
|
+
}
|
|
70
|
+
nextIndex += 1;
|
|
71
|
+
}
|
|
72
|
+
return {
|
|
73
|
+
...head,
|
|
74
|
+
blocks,
|
|
75
|
+
...(nextOffset !== undefined || nextIndex < manifest.blocks.length
|
|
76
|
+
? {
|
|
77
|
+
nextCursor: encodeDataBlockCursor({
|
|
78
|
+
scope,
|
|
79
|
+
collectedAt,
|
|
80
|
+
blockIndex: nextIndex,
|
|
81
|
+
...(nextOffset === undefined
|
|
82
|
+
? {}
|
|
83
|
+
: { intraBlockOffset: nextOffset }),
|
|
84
|
+
...(options.cursorView ? { view: options.cursorView } : {}),
|
|
85
|
+
}),
|
|
86
|
+
}
|
|
87
|
+
: {}),
|
|
88
|
+
warnings: manifest.warnings,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
function pageBlock(block, offsetBytes, maxBytes) {
|
|
92
|
+
const text = typeof block.value === "string" ? block.value : JSON.stringify(block.value);
|
|
93
|
+
const bytes = textEncoder.encode(text);
|
|
94
|
+
if (offsetBytes <= 0 && bytes.length <= maxBytes) {
|
|
95
|
+
return { block };
|
|
96
|
+
}
|
|
97
|
+
const start = Math.min(Math.max(0, offsetBytes), bytes.length);
|
|
98
|
+
const end = Math.min(bytes.length, start + Math.max(1, maxBytes));
|
|
99
|
+
return {
|
|
100
|
+
block: {
|
|
101
|
+
...block,
|
|
102
|
+
path: `${block.path}[bytes ${start}:${end}]`,
|
|
103
|
+
mediaType: block.mediaType.startsWith("text/")
|
|
104
|
+
? block.mediaType
|
|
105
|
+
: TEXT_PAGE_MEDIA_TYPE,
|
|
106
|
+
value: textDecoder.decode(bytes.slice(start, end)),
|
|
107
|
+
sizeBytes: end - start,
|
|
108
|
+
truncated: end < bytes.length,
|
|
109
|
+
},
|
|
110
|
+
...(end < bytes.length ? { nextOffset: end } : {}),
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
//# sourceMappingURL=page.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"page.js","sourceRoot":"","sources":["../../../src/storage/blocks/page.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,qBAAqB,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AAC7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AAOrD,MAAM,oBAAoB,GAAG,2BAA2B,CAAC;AACzD,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;AACtC,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;AAEtC,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,KAAgE,EAChE,OAMC;IAED,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,KAAK,CAAC;IAChD,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,QAAQ,CAAC;IACxC,MAAM,IAAI,GAAG;QACX,KAAK;QACL,WAAW;QACX,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,WAAW,EAAE,QAAQ,CAAC,WAAW;KAClC,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IAEpE,IAAI,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,MAAM,sBAAsB,CAC5C,QAAQ,EACR,OAAO,CAAC,QAAQ,EAChB,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,EAC9B,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,CAC7C,CAAC;QACF,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,SAAS,CAAC,MAAM;YACxB,QAAQ,EAAE,CAAC,GAAG,QAAQ,CAAC,QAAQ,EAAE,GAAG,SAAS,CAAC,QAAQ,CAAC;SACxD,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM;QACjC,CAAC,CAAC,uBAAuB,CAAC,OAAO,CAAC,MAAM,EAAE;YACtC,KAAK;YACL,WAAW;YACX,IAAI,EAAE,OAAO,CAAC,UAAU;SACzB,CAAC;QACJ,CAAC,CAAC,EAAE,EAAE,EAAE,IAAa,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACxC,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC;QACrB,MAAM,IAAI,qBAAqB,CAC7B,gBAAgB,EAChB,YAAY,CAAC,KAAK,CAAC,OAAO,CAC3B,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,EAAE,UAAU,IAAI,CAAC,CAAC;IACxD,MAAM,WAAW,GAAG,YAAY,CAAC,MAAM,EAAE,gBAAgB,IAAI,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAqB,EAAE,CAAC;IACpC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,SAAS,GAAG,UAAU,CAAC;IAC3B,IAAI,UAA8B,CAAC;IAEnC,OAAO,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC1C,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACvC,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACjD,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;YACnB,SAAS,IAAI,CAAC,CAAC;YACf,SAAS;QACX,CAAC;QACD,MAAM,MAAM,GAAG,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1D,IAAI,MAAM,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;YAC5B,SAAS,IAAI,CAAC,CAAC;YACf,SAAS;QACX,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,KAAK,CAAC,IAAI,KAAK,GAAG,GAAG,CAAC,SAAS,GAAG,QAAQ,EAAE,CAAC;YAC1E,MAAM;QACR,CAAC;QACD,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,GAAG,KAAK,CAAC,CAAC;QACxD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACxB,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;QAC9B,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAClC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;YAC7B,MAAM;QACR,CAAC;QACD,SAAS,IAAI,CAAC,CAAC;IACjB,CAAC;IAED,OAAO;QACL,GAAG,IAAI;QACP,MAAM;QACN,GAAG,CAAC,UAAU,KAAK,SAAS,IAAI,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM;YAChE,CAAC,CAAC;gBACE,UAAU,EAAE,qBAAqB,CAAC;oBAChC,KAAK;oBACL,WAAW;oBACX,UAAU,EAAE,SAAS;oBACrB,GAAG,CAAC,UAAU,KAAK,SAAS;wBAC1B,CAAC,CAAC,EAAE;wBACJ,CAAC,CAAC,EAAE,gBAAgB,EAAE,UAAU,EAAE,CAAC;oBACrC,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAC5D,CAAC;aACH;YACH,CAAC,CAAC,EAAE,CAAC;QACP,QAAQ,EAAE,QAAQ,CAAC,QAAQ;KAC5B,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAChB,KAAqB,EACrB,WAAmB,EACnB,QAAgB;IAEhB,MAAM,IAAI,GACR,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC9E,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,WAAW,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;QACjD,OAAO,EAAE,KAAK,EAAE,CAAC;IACnB,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;IAClE,OAAO;QACL,KAAK,EAAE;YACL,GAAG,KAAK;YACR,IAAI,EAAE,GAAG,KAAK,CAAC,IAAI,UAAU,KAAK,IAAI,GAAG,GAAG;YAC5C,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC;gBAC5C,CAAC,CAAC,KAAK,CAAC,SAAS;gBACjB,CAAC,CAAC,oBAAoB;YACxB,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAClD,SAAS,EAAE,GAAG,GAAG,KAAK;YACtB,SAAS,EAAE,GAAG,GAAG,KAAK,CAAC,MAAM;SAC9B;QACD,GAAG,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACnD,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Serve legacy scope bodies from stored PDPP records.
|
|
3
|
+
*
|
|
4
|
+
* PDPP connectors store one envelope per stream, body `{records: [row, …]}`,
|
|
5
|
+
* under `<source>.<stream>`. Apps that predate PDPP read legacy bodies such
|
|
6
|
+
* as `chatgpt.conversations` = `{conversations, total}`. `withLegacyProjection`
|
|
7
|
+
* wraps the raw storage port: when the latest stored body of a projected
|
|
8
|
+
* scope is `{records}`, reads return the legacy body projected from that
|
|
9
|
+
* stream and its sibling streams. Every other body is served exactly as
|
|
10
|
+
* stored. Writes, listings and the sync-facing methods pass through.
|
|
11
|
+
*
|
|
12
|
+
* Give the wrapped ("served") port to every reader of data: the HTTP data
|
|
13
|
+
* route, MCP, derivatives and the job worker. Give the raw port to sync
|
|
14
|
+
* upload/download, so canonical bytes (and their hashes) never change.
|
|
15
|
+
*/
|
|
16
|
+
import { type ProjectionDiagnostic, type ProjectionError } from "../legacy-projection/index.js";
|
|
17
|
+
import type { DataStoragePort } from "../ports/index.js";
|
|
18
|
+
/**
|
|
19
|
+
* - `legacy`: anything that is not exactly `{records: [...]}`. Served as stored.
|
|
20
|
+
* - `pdpp-projected`: `{records}` of plain rows under a scope with a legacy
|
|
21
|
+
* binding of a projected source. Served as the projected legacy body.
|
|
22
|
+
* - `pdpp-unprojected`: `{records}` under any other scope (for example
|
|
23
|
+
* `chatgpt.messages`, which has no legacy form), or `{records}` holding a
|
|
24
|
+
* tagged `{stream, data}` row, which is not one stream's rows. Served as
|
|
25
|
+
* stored.
|
|
26
|
+
*/
|
|
27
|
+
export type StoredBodyForm = "legacy" | "pdpp-projected" | "pdpp-unprojected";
|
|
28
|
+
export declare function classifyStoredBody(scope: string, data: unknown): StoredBodyForm;
|
|
29
|
+
export type LegacyProjectionIssue = {
|
|
30
|
+
kind: "projection_failed";
|
|
31
|
+
scope: string;
|
|
32
|
+
collectedAt: string;
|
|
33
|
+
error: ProjectionError;
|
|
34
|
+
} | {
|
|
35
|
+
kind: "projection_diagnostics";
|
|
36
|
+
scope: string;
|
|
37
|
+
collectedAt: string;
|
|
38
|
+
diagnostics: ProjectionDiagnostic[];
|
|
39
|
+
};
|
|
40
|
+
export interface LegacyProjectionOptions {
|
|
41
|
+
/**
|
|
42
|
+
* Called when a projection fails (the stored body is then served as
|
|
43
|
+
* stored) or succeeds with dropped rows or a missing join stream.
|
|
44
|
+
*/
|
|
45
|
+
onIssue?: (issue: LegacyProjectionIssue) => void;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Wrap a raw storage port so reads of projected scopes return legacy bodies.
|
|
49
|
+
* Wrapping an already wrapped port returns it unchanged.
|
|
50
|
+
*/
|
|
51
|
+
export declare function withLegacyProjection(raw: DataStoragePort, options?: LegacyProjectionOptions): DataStoragePort;
|
|
52
|
+
/**
|
|
53
|
+
* The projected legacy scopes whose served body is built from `scope`'s
|
|
54
|
+
* stored stream, other than `scope` itself: `chatgpt.messages` →
|
|
55
|
+
* `["chatgpt.conversations"]`. Anything that caches a read of a projected
|
|
56
|
+
* scope must treat a change to these streams as a change to it.
|
|
57
|
+
*/
|
|
58
|
+
export declare function legacyScopesProjectedFrom(scope: string): string[];
|
|
59
|
+
export interface SupersedeResult {
|
|
60
|
+
/** `collectedAt` of every deleted version. */
|
|
61
|
+
superseded: string[];
|
|
62
|
+
/** Why nothing was deleted, when superseding was refused. */
|
|
63
|
+
refused?: string;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* After a `{records}` version of a projected scope is committed, delete the
|
|
67
|
+
* scope's older owner-written legacy versions: the new version serves the
|
|
68
|
+
* same legacy view, so they are a second copy.
|
|
69
|
+
*
|
|
70
|
+
* It deletes nothing unless `storage` is a served port (see
|
|
71
|
+
* `withLegacyProjection`) whose projection of the new version is complete:
|
|
72
|
+
* no join stream missing and no row left out. It also deletes nothing when
|
|
73
|
+
* the projection holds fewer items than the newest legacy version, or lacks
|
|
74
|
+
* any item id that version holds, so an empty or time-windowed run cannot
|
|
75
|
+
* replace a full history. For Claude scopes it also deletes nothing when an
|
|
76
|
+
* item holds fewer nested messages or documents than the legacy item. A legacy version must carry the scope's legacy
|
|
77
|
+
* collection key (`conversations` for `chatgpt.conversations`); PDPP
|
|
78
|
+
* versions, other bodies, binary uploads, versions a builder wrote
|
|
79
|
+
* (`$writtenBy`) and newer versions are kept.
|
|
80
|
+
*
|
|
81
|
+
* Local only: the deletes are not sent to the gateway. Copies that sync
|
|
82
|
+
* already uploaded, and copies other replicas already downloaded, remain.
|
|
83
|
+
*/
|
|
84
|
+
export declare function supersedeLegacyVersions(storage: DataStoragePort, scope: string, committedAt: string): Promise<SupersedeResult>;
|
|
85
|
+
//# sourceMappingURL=legacy-projection.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"legacy-projection.d.ts","sourceRoot":"","sources":["../../src/storage/legacy-projection.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,EAIL,KAAK,oBAAoB,EACzB,KAAK,eAAe,EACrB,MAAM,+BAA+B,CAAC;AACvC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAqBzD;;;;;;;;GAQG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,gBAAgB,GAAG,kBAAkB,CAAC;AAE9E,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,OAAO,GACZ,cAAc,CAKhB;AAED,MAAM,MAAM,qBAAqB,GAC7B;IACE,IAAI,EAAE,mBAAmB,CAAC;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,eAAe,CAAC;CACxB,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,oBAAoB,EAAE,CAAC;CACrC,CAAC;AAEN,MAAM,WAAW,uBAAuB;IACtC;;;OAGG;IACH,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,qBAAqB,KAAK,IAAI,CAAC;CAClD;AAwBD;;;GAGG;AACH,wBAAgB,oBAAoB,CAClC,GAAG,EAAE,eAAe,EACpB,OAAO,GAAE,uBAA4B,GACpC,eAAe,CAgQjB;AAED;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,CAcjE;AAwGD,MAAM,WAAW,eAAe;IAC9B,8CAA8C;IAC9C,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,6DAA6D;IAC7D,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,uBAAuB,CAC3C,OAAO,EAAE,eAAe,EACxB,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,eAAe,CAAC,CA+F1B"}
|