@deveye/types 0.20.2 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/domain/admission.ts +37 -0
- package/src/domain/device.ts +15 -16
- package/src/domain/featureRegistry.ts +3 -2
- package/src/domain/maintenance.ts +7 -1
- package/src/domain/notifications.ts +16 -0
- package/src/domain/packages.ts +34 -2
- package/src/domain/report.ts +12 -3
- package/src/domain/secrecy.ts +13 -0
- package/src/domain/user.ts +2 -0
- package/src/features/admin.ts +59 -1
- package/src/features/debug.ts +372 -0
- package/src/features/registry.ts +2 -0
- package/src/features/user.ts +46 -0
- package/src/http/device.ts +34 -30
- package/src/http/status.ts +6 -0
- package/src/index.ts +105 -10
- package/src/protocol/agent.ts +35 -2
- package/src/protocol/error.ts +4 -1
- package/src/sdk/accountExport.test.ts +115 -0
- package/src/sdk/accountExport.ts +254 -0
- package/src/sdk/client-ambient.d.ts +86 -0
- package/src/sdk/client.ts +44 -0
- package/src/sdk/devb.test.ts +148 -0
- package/src/sdk/devb.ts +146 -0
- package/src/sdk/download.test.ts +40 -0
- package/src/sdk/download.ts +48 -0
- package/src/sdk/manifest.test.ts +17 -1
- package/src/sdk/manifest.ts +51 -11
- package/src/sdk/net.test.ts +42 -1
- package/src/sdk/net.ts +100 -2
- package/src/sdk/pool.test.ts +36 -0
- package/src/sdk/pool.ts +26 -0
- package/src/sdk/providers.ts +107 -11
- package/src/sdk/server.ts +392 -20
- package/src/sdk/testing.test.ts +66 -0
- package/src/sdk/testing.ts +211 -3
package/src/index.ts
CHANGED
|
@@ -9,6 +9,7 @@ export type { ProtocolVersion } from './protocol/version';
|
|
|
9
9
|
export {
|
|
10
10
|
AGENT_ACK,
|
|
11
11
|
AGENT_AUTH_EVENTS,
|
|
12
|
+
AGENT_CLOSE_PENDING_APPROVAL,
|
|
12
13
|
AGENT_COLLECT,
|
|
13
14
|
AGENT_CONFIG,
|
|
14
15
|
AGENT_DESTROY,
|
|
@@ -71,6 +72,7 @@ export {
|
|
|
71
72
|
AGENT_SYNC_INDEX,
|
|
72
73
|
AGENT_SYNC_OP_RESULT,
|
|
73
74
|
AGENT_SYNC_PUSH,
|
|
75
|
+
AGENT_SYNC_PUSH_ACK,
|
|
74
76
|
AGENT_SYNC_SCAN,
|
|
75
77
|
AGENT_TERM_CLOSE,
|
|
76
78
|
AGENT_TERM_EXIT,
|
|
@@ -137,6 +139,7 @@ export {
|
|
|
137
139
|
agentSyncIndexPayloadSchema,
|
|
138
140
|
agentSyncOpResultPayloadSchema,
|
|
139
141
|
agentSyncPushPayloadSchema,
|
|
142
|
+
agentSyncPushAckPayloadSchema,
|
|
140
143
|
agentSyncScanPayloadSchema,
|
|
141
144
|
agentTermClosePayloadSchema,
|
|
142
145
|
agentTermExitPayloadSchema,
|
|
@@ -242,6 +245,7 @@ export type {
|
|
|
242
245
|
AgentSyncMovePayload,
|
|
243
246
|
AgentSyncOpResultPayload,
|
|
244
247
|
AgentSyncPushPayload,
|
|
248
|
+
AgentSyncPushAckPayload,
|
|
245
249
|
AgentSyncScanPayload,
|
|
246
250
|
AgentTermClosePayload,
|
|
247
251
|
AgentTermInputPayload,
|
|
@@ -312,6 +316,15 @@ export type {
|
|
|
312
316
|
FeedbackStatus,
|
|
313
317
|
FeedbackViewTrace
|
|
314
318
|
} from './domain/feedback';
|
|
319
|
+
export {
|
|
320
|
+
IDLE_CLOSE_CODE,
|
|
321
|
+
QUEUE_CLOSE_CODE,
|
|
322
|
+
queueRefusalSchema,
|
|
323
|
+
seatCapsSchema,
|
|
324
|
+
seatSchema,
|
|
325
|
+
SESSION_ACTIVE_COMMAND
|
|
326
|
+
} from './domain/admission';
|
|
327
|
+
export type { QueueRefusal, Seat, SeatCaps } from './domain/admission';
|
|
315
328
|
export {
|
|
316
329
|
featureMaintenanceLevelSchema,
|
|
317
330
|
MAINTENANCE_CLOSE_CODE,
|
|
@@ -561,8 +574,18 @@ export type {
|
|
|
561
574
|
UserSettingFlag,
|
|
562
575
|
UserStatus
|
|
563
576
|
} from './domain/user';
|
|
564
|
-
export {
|
|
565
|
-
|
|
577
|
+
export {
|
|
578
|
+
SECRECY_STATE_EVENT,
|
|
579
|
+
secrecyStatePushSchema,
|
|
580
|
+
secrecyStatusSchema,
|
|
581
|
+
secrecyWrapModeSchema
|
|
582
|
+
} from './domain/secrecy';
|
|
583
|
+
export type {
|
|
584
|
+
SecrecyStatePush,
|
|
585
|
+
SecrecyStatus,
|
|
586
|
+
SecrecyWrapMode,
|
|
587
|
+
UserSecretKeyRow
|
|
588
|
+
} from './domain/secrecy';
|
|
566
589
|
export { workspaceKindSchema, workspaceSchema } from './domain/workspace';
|
|
567
590
|
export {
|
|
568
591
|
REMOTE_INSTANCES_MAX,
|
|
@@ -615,8 +638,13 @@ export {
|
|
|
615
638
|
terminalUser
|
|
616
639
|
} from './domain/device';
|
|
617
640
|
export type { Device, DevicePlatform, DeviceRow, DeviceStatus } from './domain/device';
|
|
618
|
-
export {
|
|
619
|
-
|
|
641
|
+
export {
|
|
642
|
+
isManagedPackageManager,
|
|
643
|
+
packageManagerIdSchema,
|
|
644
|
+
packageManagerSchema,
|
|
645
|
+
unmanagedUpdaterIdSchema
|
|
646
|
+
} from './domain/packages';
|
|
647
|
+
export type { PackageManager, PackageManagerId, UnmanagedUpdaterId } from './domain/packages';
|
|
620
648
|
export {
|
|
621
649
|
agentInfoSchema,
|
|
622
650
|
agentPolicySchema,
|
|
@@ -673,6 +701,9 @@ export {
|
|
|
673
701
|
NOTIFICATION_EMAIL_MAX,
|
|
674
702
|
NOTIFICATION_LABEL_MAX,
|
|
675
703
|
NOTIFICATION_TARGET_MAX,
|
|
704
|
+
SYSTEM_NOTIFICATION_INFO,
|
|
705
|
+
SYSTEM_NOTIFICATION_TARGET,
|
|
706
|
+
systemNotificationTargetSchema,
|
|
676
707
|
notificationChannelInputSchema,
|
|
677
708
|
notificationChannelKindSchema,
|
|
678
709
|
notificationChannelSchema,
|
|
@@ -696,7 +727,8 @@ export type {
|
|
|
696
727
|
NotificationRouteInput,
|
|
697
728
|
NotificationRouteRow,
|
|
698
729
|
NotificationRouteTarget,
|
|
699
|
-
NotificationTest
|
|
730
|
+
NotificationTest,
|
|
731
|
+
SystemNotificationTarget
|
|
700
732
|
} from './domain/notifications';
|
|
701
733
|
|
|
702
734
|
export {
|
|
@@ -824,7 +856,10 @@ export {
|
|
|
824
856
|
adminMaintenanceDismissNotice,
|
|
825
857
|
adminMaintenanceFeature,
|
|
826
858
|
adminMaintenanceGet,
|
|
859
|
+
adminMaintenancePriority,
|
|
827
860
|
adminMaintenanceSchema,
|
|
861
|
+
adminMaintenanceSeats,
|
|
862
|
+
adminMaintenanceSignups,
|
|
828
863
|
adminMaintenanceSite,
|
|
829
864
|
adminSetUserRole,
|
|
830
865
|
adminSetUserStatus,
|
|
@@ -832,6 +867,64 @@ export {
|
|
|
832
867
|
adminUserSchema
|
|
833
868
|
} from './features/admin';
|
|
834
869
|
export type { AdminMaintenance, AdminUser } from './features/admin';
|
|
870
|
+
export {
|
|
871
|
+
debugBenchCatalog,
|
|
872
|
+
debugBenchContextSchema,
|
|
873
|
+
debugBenchProfileSchema,
|
|
874
|
+
debugBenchReportSchema,
|
|
875
|
+
debugBenchStart,
|
|
876
|
+
debugCommands,
|
|
877
|
+
debugE2eCatalog,
|
|
878
|
+
debugE2eReportSchema,
|
|
879
|
+
debugE2eScenarioReportSchema,
|
|
880
|
+
debugE2eScenarioSchema,
|
|
881
|
+
debugE2eStart,
|
|
882
|
+
debugE2eStepSchema,
|
|
883
|
+
debugE2eSweep,
|
|
884
|
+
debugInstanceSchema,
|
|
885
|
+
debugMailCatalog,
|
|
886
|
+
debugMailPreview,
|
|
887
|
+
debugMailPreviewSchema,
|
|
888
|
+
debugMailSampleSchema,
|
|
889
|
+
debugMailSend,
|
|
890
|
+
debugMailSenderKindSchema,
|
|
891
|
+
debugOverview,
|
|
892
|
+
debugPing,
|
|
893
|
+
debugProbeResultSchema,
|
|
894
|
+
debugProbeSchema,
|
|
895
|
+
debugRunAbort,
|
|
896
|
+
debugRunGet,
|
|
897
|
+
debugRunKindSchema,
|
|
898
|
+
debugRunList,
|
|
899
|
+
debugRunSchema,
|
|
900
|
+
debugRunStatusSchema,
|
|
901
|
+
debugStepStatusSchema,
|
|
902
|
+
debugTrackingClear,
|
|
903
|
+
debugTrackingCreate,
|
|
904
|
+
debugTrackingGet,
|
|
905
|
+
debugTrackingSchema,
|
|
906
|
+
debugTrackingSet,
|
|
907
|
+
debugTrackingUse,
|
|
908
|
+
debugWorkspaceSenderSchema
|
|
909
|
+
} from './features/debug';
|
|
910
|
+
export type {
|
|
911
|
+
DebugBenchContext,
|
|
912
|
+
DebugBenchProfile,
|
|
913
|
+
DebugBenchReport,
|
|
914
|
+
DebugE2eReport,
|
|
915
|
+
DebugE2eScenario,
|
|
916
|
+
DebugE2eScenarioReport,
|
|
917
|
+
DebugE2eStep,
|
|
918
|
+
DebugMailPreview,
|
|
919
|
+
DebugMailSample,
|
|
920
|
+
DebugProbe,
|
|
921
|
+
DebugProbeResult,
|
|
922
|
+
DebugRun,
|
|
923
|
+
DebugRunKind,
|
|
924
|
+
DebugStepStatus,
|
|
925
|
+
DebugTracking,
|
|
926
|
+
DebugWorkspaceSender
|
|
927
|
+
} from './features/debug';
|
|
835
928
|
export {
|
|
836
929
|
workspaceActivate,
|
|
837
930
|
workspaceAdd,
|
|
@@ -866,6 +959,9 @@ export {
|
|
|
866
959
|
accountPlanSchema,
|
|
867
960
|
userCommands,
|
|
868
961
|
userDeleteAccount,
|
|
962
|
+
userExportPartSchema,
|
|
963
|
+
userExportPrepare,
|
|
964
|
+
userExportPreview,
|
|
869
965
|
userPlan,
|
|
870
966
|
userSetAvatar,
|
|
871
967
|
userSetColor,
|
|
@@ -873,7 +969,7 @@ export {
|
|
|
873
969
|
userSetTheme,
|
|
874
970
|
userSetUsername
|
|
875
971
|
} from './features/user';
|
|
876
|
-
export type { ThemeStateDTO } from './features/user';
|
|
972
|
+
export type { ThemeStateDTO, UserExportPart } from './features/user';
|
|
877
973
|
export {
|
|
878
974
|
agentCollect,
|
|
879
975
|
agentCommands,
|
|
@@ -1012,11 +1108,11 @@ export {
|
|
|
1012
1108
|
agentTargetStatusSchema,
|
|
1013
1109
|
enrollDeviceRequestSchema,
|
|
1014
1110
|
enrollDeviceResponseSchema,
|
|
1111
|
+
LINK_CODE_MAX_USES,
|
|
1015
1112
|
LINK_CODE_TTL_MAX_SECONDS,
|
|
1016
1113
|
linkCodeRequestSchema,
|
|
1017
1114
|
linkCodeResponseSchema,
|
|
1018
|
-
linkCodesListResponseSchema
|
|
1019
|
-
linkCodeUpdateSchema
|
|
1115
|
+
linkCodesListResponseSchema
|
|
1020
1116
|
} from './http/device';
|
|
1021
1117
|
export type {
|
|
1022
1118
|
AgentManifest,
|
|
@@ -1030,8 +1126,7 @@ export type {
|
|
|
1030
1126
|
EnrollDeviceResponse,
|
|
1031
1127
|
LinkCodeRequest,
|
|
1032
1128
|
LinkCodeResponse,
|
|
1033
|
-
LinkCodesListResponse
|
|
1034
|
-
LinkCodeUpdate
|
|
1129
|
+
LinkCodesListResponse
|
|
1035
1130
|
} from './http/device';
|
|
1036
1131
|
export {
|
|
1037
1132
|
BUILD_MANIFEST_PATH,
|
package/src/protocol/agent.ts
CHANGED
|
@@ -418,7 +418,13 @@ export const agentSyncChunkPayloadSchema = z.object({
|
|
|
418
418
|
hash: sha256HexSchema.optional(),
|
|
419
419
|
size: z.number().int().nonnegative().optional(),
|
|
420
420
|
mtime: z.number().int().nonnegative().optional(),
|
|
421
|
-
error: z.string().max(500).optional()
|
|
421
|
+
error: z.string().max(500).optional(),
|
|
422
|
+
/**
|
|
423
|
+
* Rank of the frame in its push, from 0. Present only when the agent
|
|
424
|
+
* honours the push's `window`: the server then answers each data frame with
|
|
425
|
+
* `sync.pushAck`. Absent (an older agent), nothing is acknowledged.
|
|
426
|
+
*/
|
|
427
|
+
seq: z.number().int().nonnegative().optional()
|
|
422
428
|
});
|
|
423
429
|
export type AgentSyncChunkPayload = z.infer<typeof agentSyncChunkPayloadSchema>;
|
|
424
430
|
|
|
@@ -640,6 +646,14 @@ export const AGENT_CONFIG = 'agent.config' as const;
|
|
|
640
646
|
export const AGENT_TOKEN_ROTATE = 'agent.tokenRotate' as const;
|
|
641
647
|
export const agentTokenRotatePayloadSchema = z.object({ token: z.string().min(1).max(4096) });
|
|
642
648
|
export type AgentTokenRotatePayload = z.infer<typeof agentTokenRotatePayloadSchema>;
|
|
649
|
+
/**
|
|
650
|
+
* WebSocket close code for an authenticated agent whose device waits for
|
|
651
|
+
* approval: it retries on a short fixed delay instead of backing off like a
|
|
652
|
+
* refused one. Every refusal (unknown, invalid or archived) stays `1008`, so
|
|
653
|
+
* only a holder of a valid token learns the device is pending. Mirrored by
|
|
654
|
+
* `CLOSE_PENDING_APPROVAL` in the agent's `runner.rs`.
|
|
655
|
+
*/
|
|
656
|
+
export const AGENT_CLOSE_PENDING_APPROVAL = 4001;
|
|
643
657
|
/** Tell the agent to self-destruct (wipe its local config + binary) and exit. */
|
|
644
658
|
export const AGENT_DESTROY = 'agent.destroy' as const;
|
|
645
659
|
/** Tell the agent to download, verify and swap in a newer signed binary. */
|
|
@@ -914,10 +928,25 @@ export const agentSyncPushPayloadSchema = z.object({
|
|
|
914
928
|
opId: syncOpId,
|
|
915
929
|
shareId: z.number().int().positive(),
|
|
916
930
|
relPath: syncRelPath,
|
|
917
|
-
startOffset: z.number().int().nonnegative().default(0)
|
|
931
|
+
startOffset: z.number().int().nonnegative().default(0),
|
|
932
|
+
/**
|
|
933
|
+
* Data frames the agent may have unacknowledged at once: it numbers its
|
|
934
|
+
* frames (`seq`) and waits for a `sync.pushAck` before going past the
|
|
935
|
+
* window. The server acks a frame once written, and paces its acks to cap
|
|
936
|
+
* the upload rate. Absent (an older server), the agent streams freely.
|
|
937
|
+
*/
|
|
938
|
+
window: z.number().int().positive().max(64).optional()
|
|
918
939
|
});
|
|
919
940
|
export type AgentSyncPushPayload = z.infer<typeof agentSyncPushPayloadSchema>;
|
|
920
941
|
|
|
942
|
+
/** One credit back for a windowed push: data frame `seq` of `opId` is written. */
|
|
943
|
+
export const AGENT_SYNC_PUSH_ACK = 'sync.pushAck' as const;
|
|
944
|
+
export const agentSyncPushAckPayloadSchema = z.object({
|
|
945
|
+
opId: syncOpId,
|
|
946
|
+
seq: z.number().int().nonnegative()
|
|
947
|
+
});
|
|
948
|
+
export type AgentSyncPushAckPayload = z.infer<typeof agentSyncPushAckPayloadSchema>;
|
|
949
|
+
|
|
921
950
|
/**
|
|
922
951
|
* Un chunk de download à installer. hash/size/mtime sont répétés sur chaque
|
|
923
952
|
* frame (sans état) ; l'agent écrit dans un fichier temporaire, vérifie le
|
|
@@ -1200,6 +1229,10 @@ export const agentServerMessageSchema = z.discriminatedUnion('command', [
|
|
|
1200
1229
|
command: z.literal(AGENT_SYNC_PUSH),
|
|
1201
1230
|
payload: agentSyncPushPayloadSchema
|
|
1202
1231
|
}),
|
|
1232
|
+
z.object({
|
|
1233
|
+
command: z.literal(AGENT_SYNC_PUSH_ACK),
|
|
1234
|
+
payload: agentSyncPushAckPayloadSchema
|
|
1235
|
+
}),
|
|
1203
1236
|
z.object({
|
|
1204
1237
|
command: z.literal(AGENT_SYNC_APPLY_CHUNK),
|
|
1205
1238
|
payload: agentSyncApplyChunkPayloadSchema
|
package/src/protocol/error.ts
CHANGED
|
@@ -19,7 +19,10 @@ export const ErrorCodeSchema = z.enum([
|
|
|
19
19
|
'locked',
|
|
20
20
|
// The site, or the feature the command belongs to, is under maintenance
|
|
21
21
|
// (see `domain/maintenance.ts`). The message is the one to show.
|
|
22
|
-
'maintenance'
|
|
22
|
+
'maintenance',
|
|
23
|
+
// The server is full for this kind of account: the socket waits in line
|
|
24
|
+
// (see `domain/admission.ts`). `details` carries `{ position }`.
|
|
25
|
+
'queued'
|
|
23
26
|
]);
|
|
24
27
|
|
|
25
28
|
export type ErrorCode = z.infer<typeof ErrorCodeSchema>;
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import { test } from 'node:test';
|
|
3
|
+
|
|
4
|
+
import { accountExportProblem, exportTableRows, type SdkExportTable } from './accountExport';
|
|
5
|
+
import type { SdkQueryable } from './server';
|
|
6
|
+
|
|
7
|
+
const spec: SdkExportTable = { file: 'notes.json', where: 'workspace_id = ?', key: ['id'] };
|
|
8
|
+
|
|
9
|
+
test('accountExportProblem accepts a sound declaration and names what is wrong', () => {
|
|
10
|
+
assert.equal(
|
|
11
|
+
accountExportProblem({
|
|
12
|
+
tables: {
|
|
13
|
+
notes: { ...spec, sealed: ['content'], json: ['content'], omit: ['secret_enc'] },
|
|
14
|
+
note_cache: { skip: 'Un cache, reconstruit par la synchronisation.' },
|
|
15
|
+
notes_links: 'custom'
|
|
16
|
+
},
|
|
17
|
+
store: { omit: ['apiKey'] }
|
|
18
|
+
}),
|
|
19
|
+
null
|
|
20
|
+
);
|
|
21
|
+
const problem = (tables: Record<string, unknown>) =>
|
|
22
|
+
accountExportProblem({ tables: tables as never });
|
|
23
|
+
assert.match(problem({ 'bad name': spec }) ?? '', /not an identifier/);
|
|
24
|
+
assert.match(problem({ a: { skip: ' ' } }) ?? '', /a skip says why/);
|
|
25
|
+
assert.match(problem({ a: { ...spec, file: 'Notes.JSON' } }) ?? '', /file/);
|
|
26
|
+
assert.match(problem({ a: spec, b: spec }) ?? '', /write the same file/);
|
|
27
|
+
assert.equal(problem({ a: spec, b: { ...spec, scope: 'account' } }), null);
|
|
28
|
+
assert.match(problem({ a: { ...spec, where: 'a = ? AND b = ?' } }) ?? '', /exactly one \?/);
|
|
29
|
+
assert.match(problem({ a: { ...spec, where: 'a = ?; DROP TABLE x' } }) ?? '', /holds a ;/);
|
|
30
|
+
assert.match(
|
|
31
|
+
problem({ a: { ...spec, sealed: ['x'], omit: ['x'] } }) ?? '',
|
|
32
|
+
/both sealed and omitted/
|
|
33
|
+
);
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
function fakeTable(rows: Record<string, unknown>[]): { q: SdkQueryable; sql: string[] } {
|
|
37
|
+
const sql: string[] = [];
|
|
38
|
+
const q: SdkQueryable = {
|
|
39
|
+
async query<T extends object>(text: string, params: unknown[] = []): Promise<T[]> {
|
|
40
|
+
sql.push(text);
|
|
41
|
+
const after = params.length > 1 ? Number(params[1]) : -Infinity;
|
|
42
|
+
const limit = Number(/LIMIT (\d+)/.exec(text)?.[1]);
|
|
43
|
+
return rows.filter((r) => Number(r.id) > after).slice(0, limit) as T[];
|
|
44
|
+
},
|
|
45
|
+
async execute() {
|
|
46
|
+
return { affectedRows: 0, insertId: 0 };
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
return { q, sql };
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
test('exportTableRows pages by key, opens sealed cells, and never lets an omitted column out', async () => {
|
|
53
|
+
const rows = Array.from({ length: 450 }, (_, i) => ({
|
|
54
|
+
id: i + 1,
|
|
55
|
+
workspace_id: 3,
|
|
56
|
+
content: i === 7 ? 'broken' : `sealed:${JSON.stringify({ n: i })}`,
|
|
57
|
+
secret_enc: 'nope',
|
|
58
|
+
created: 1_790_000_000,
|
|
59
|
+
big: BigInt('12345678901234567890'),
|
|
60
|
+
raw: Buffer.from('hi')
|
|
61
|
+
}));
|
|
62
|
+
const { q, sql } = fakeTable(rows);
|
|
63
|
+
let unreadable = 0;
|
|
64
|
+
const out: Record<string, unknown>[] = [];
|
|
65
|
+
for await (const row of exportTableRows(
|
|
66
|
+
q,
|
|
67
|
+
'notes',
|
|
68
|
+
{
|
|
69
|
+
...spec,
|
|
70
|
+
sealed: ['content'],
|
|
71
|
+
json: ['content'],
|
|
72
|
+
omit: ['secret_enc'],
|
|
73
|
+
dates: { created: 's' }
|
|
74
|
+
},
|
|
75
|
+
3,
|
|
76
|
+
async (blob) => (blob.startsWith('sealed:') ? blob.slice(7) : null),
|
|
77
|
+
() => unreadable++
|
|
78
|
+
)) {
|
|
79
|
+
out.push(row);
|
|
80
|
+
}
|
|
81
|
+
assert.equal(out.length, 450);
|
|
82
|
+
assert.deepEqual(out[0], {
|
|
83
|
+
id: 1,
|
|
84
|
+
workspace_id: 3,
|
|
85
|
+
content: { n: 0 },
|
|
86
|
+
created: new Date(1_790_000_000_000).toISOString(),
|
|
87
|
+
big: '12345678901234567890',
|
|
88
|
+
raw: { base64: Buffer.from('hi').toString('base64') }
|
|
89
|
+
});
|
|
90
|
+
assert.equal(out[7].content, null);
|
|
91
|
+
assert.equal(unreadable, 1);
|
|
92
|
+
assert.ok(sql.length >= 2, 'several pages');
|
|
93
|
+
assert.match(
|
|
94
|
+
sql[0],
|
|
95
|
+
/^SELECT \* FROM `notes` WHERE \(workspace_id = \?\) ORDER BY `id` LIMIT 200$/
|
|
96
|
+
);
|
|
97
|
+
assert.match(sql[1], /AND \(`id`\) > \(\?\)/);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
test('exportTableRows writes an opened `_enc` column without its suffix, unless the name is taken', async () => {
|
|
101
|
+
const { q } = fakeTable([
|
|
102
|
+
{ id: 1, workspace_id: 3, query_enc: 'sealed:a', title: 't', title_enc: 'sealed:b' }
|
|
103
|
+
]);
|
|
104
|
+
const out: Record<string, unknown>[] = [];
|
|
105
|
+
for await (const row of exportTableRows(
|
|
106
|
+
q,
|
|
107
|
+
'lookups',
|
|
108
|
+
{ ...spec, sealed: ['query_enc', 'title_enc'] },
|
|
109
|
+
3,
|
|
110
|
+
async (blob) => blob.slice(7)
|
|
111
|
+
)) {
|
|
112
|
+
out.push(row);
|
|
113
|
+
}
|
|
114
|
+
assert.deepEqual(out, [{ id: 1, workspace_id: 3, query: 'a', title: 't', title_enc: 'b' }]);
|
|
115
|
+
});
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
import type { SdkCipher, SdkLogger, SdkQueryable, SdkServerKeys } from './server';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The holder's data export (GDPR portability): a zip streamed to their browser,
|
|
5
|
+
* never stored. Everything in it is PLAINTEXT, private tier included: the holder
|
|
6
|
+
* typed their password for it. What never goes in: what opens a service or
|
|
7
|
+
* protects the account (tokens, API keys, integration passwords, secret URLs,
|
|
8
|
+
* hashes, key material).
|
|
9
|
+
*
|
|
10
|
+
* The host exports your `FeatureStore` itself, minus `store.omit`, and every
|
|
11
|
+
* table you declare as an {@link SdkExportTable}. Your hooks write the rest:
|
|
12
|
+
* files, or a table whose rows need shaping.
|
|
13
|
+
*/
|
|
14
|
+
export interface FeatureAccountExport<Repo = unknown> {
|
|
15
|
+
/**
|
|
16
|
+
* Every table your feature owns, and its fate: exported by the host, by
|
|
17
|
+
* your hooks (`'custom'`), or left out with the sentence the holder reads
|
|
18
|
+
* in the archive. Checked at boot against the tables that exist, and
|
|
19
|
+
* against their columns (see {@link SdkExportTable.omit}).
|
|
20
|
+
*/
|
|
21
|
+
tables: Readonly<Record<string, SdkExportTable | 'custom' | SdkExportSkip>>;
|
|
22
|
+
/** Keys of your `FeatureStore` never exported (a token, an API key). The host exports the rest. */
|
|
23
|
+
store?: { omit: readonly string[] };
|
|
24
|
+
/** The large files you carry, measured for the dialog; `optional` ones the holder may leave out. */
|
|
25
|
+
files?: Readonly<Record<string, SdkExportFiles<Repo>>>;
|
|
26
|
+
/** Once per export: what you keep by account rather than by workspace. */
|
|
27
|
+
account?(ctx: SdkAccountExportContext<Repo>): Promise<void>;
|
|
28
|
+
/** Once per workspace the export covers: the personal one and the shared ones the holder owns. */
|
|
29
|
+
workspace?(ctx: SdkWorkspaceExportContext<Repo>): Promise<void>;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Left out on purpose; `skip` is the sentence the holder reads in the archive, in French. */
|
|
33
|
+
export interface SdkExportSkip {
|
|
34
|
+
skip: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** A table the host writes for you, paged, sealed cells opened. */
|
|
38
|
+
export interface SdkExportTable {
|
|
39
|
+
/** File within your folder, ending in `.json`. */
|
|
40
|
+
file: string;
|
|
41
|
+
/** `'workspace'` (default): the `?` of `where` is the workspace id. `'account'`: the holder's user id. */
|
|
42
|
+
scope?: 'workspace' | 'account';
|
|
43
|
+
/** Selects the rows, with exactly one `?`. A literal of your module, never built from input. */
|
|
44
|
+
where: string;
|
|
45
|
+
/** A unique ordering key, for keyset paging. */
|
|
46
|
+
key: readonly [string, ...string[]];
|
|
47
|
+
/**
|
|
48
|
+
* Encrypted columns, opened whatever their tier. One ending in `_enc` is
|
|
49
|
+
* written without the suffix (`query_enc` as `query`), unless the row
|
|
50
|
+
* already has a column of that name.
|
|
51
|
+
*/
|
|
52
|
+
sealed?: readonly string[];
|
|
53
|
+
/** Columns (sealed or not) holding JSON: written as values, not strings. */
|
|
54
|
+
json?: readonly string[];
|
|
55
|
+
/** Epoch columns, written as ISO 8601. */
|
|
56
|
+
dates?: Readonly<Record<string, 's' | 'ms'>>;
|
|
57
|
+
/** Never written. A column named like a secret MUST be here unless listed in `keep`. */
|
|
58
|
+
omit?: readonly string[];
|
|
59
|
+
/** Columns the secret-name rule would drop that are not secrets (`dedup_hash`). */
|
|
60
|
+
keep?: readonly string[];
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Large files your hooks write, measured before the export so the holder knows what to expect. */
|
|
64
|
+
export interface SdkExportFiles<Repo = unknown> {
|
|
65
|
+
/** As the dialog names them, in French: `'les fichiers CloudSync'`. */
|
|
66
|
+
label: string;
|
|
67
|
+
/** The holder may leave them out of the archive. */
|
|
68
|
+
optional?: boolean;
|
|
69
|
+
/** Bytes over the holder's owned workspaces; 0 hides them from the dialog. */
|
|
70
|
+
bytes(ctx: {
|
|
71
|
+
repo: Repo;
|
|
72
|
+
q: SdkQueryable;
|
|
73
|
+
userId: number;
|
|
74
|
+
workspaceIds: readonly number[];
|
|
75
|
+
}): Promise<number>;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Where your hooks write, rooted at your folder in the archive. Paths are made safe by the host. */
|
|
79
|
+
export interface SdkExportWriter {
|
|
80
|
+
/** A small JSON document. */
|
|
81
|
+
json(path: string, value: unknown): Promise<void>;
|
|
82
|
+
/** Rows as a JSON array, written as they come. Writes nothing when there is none. */
|
|
83
|
+
rows(path: string, rows: AsyncIterable<unknown>): Promise<void>;
|
|
84
|
+
/** One of your tables, as a declared {@link SdkExportTable} would be written. */
|
|
85
|
+
table(table: string, spec: SdkExportTable): Promise<void>;
|
|
86
|
+
/**
|
|
87
|
+
* A file, streamed. `compress: false` for bytes already compressed (an
|
|
88
|
+
* archive, an image). `mtime` in epoch seconds.
|
|
89
|
+
*/
|
|
90
|
+
file(
|
|
91
|
+
path: string,
|
|
92
|
+
bytes: Uint8Array | AsyncIterable<Uint8Array>,
|
|
93
|
+
opts?: { mtime?: number; compress?: boolean }
|
|
94
|
+
): Promise<void>;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
interface SdkExportContextBase<Repo> {
|
|
98
|
+
repo: Repo;
|
|
99
|
+
/** Reads only. */
|
|
100
|
+
q: SdkQueryable;
|
|
101
|
+
/** The holder. */
|
|
102
|
+
userId: number;
|
|
103
|
+
out: SdkExportWriter;
|
|
104
|
+
/** Whether the holder kept `files[key]` in the archive. */
|
|
105
|
+
includes(filesKey: string): boolean;
|
|
106
|
+
keys: SdkServerKeys;
|
|
107
|
+
/** Aborted when the download stops: check it in long loops. */
|
|
108
|
+
signal: AbortSignal;
|
|
109
|
+
logger: SdkLogger;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export interface SdkAccountExportContext<Repo = unknown> extends SdkExportContextBase<Repo> {
|
|
113
|
+
/** Every workspace the export covers. */
|
|
114
|
+
workspaceIds: readonly number[];
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export interface SdkWorkspaceExportContext<Repo = unknown> extends SdkExportContextBase<Repo> {
|
|
118
|
+
workspace: { id: number; kind: 'personal' | 'shared'; name: string };
|
|
119
|
+
/** Both tiers are readable for the length of this export. */
|
|
120
|
+
cipher(mode?: 'server' | 'private'): SdkCipher;
|
|
121
|
+
/** A sealed cell opened whatever its tier; `null` (and counted in the report) when no key opens it. */
|
|
122
|
+
open(blob: string): Promise<string | null>;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** A column name that looks like a secret: {@link SdkExportTable.omit} it, or `keep` it knowingly. */
|
|
126
|
+
export const EXPORT_SECRET_COLUMN = /(secret|token|passw|_hash$|private_key|credential|wrapped)/i;
|
|
127
|
+
|
|
128
|
+
const IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
129
|
+
|
|
130
|
+
/** What is wrong with a declaration, or `null`. Checked at boot, before any table is read. */
|
|
131
|
+
export function accountExportProblem(entry: FeatureAccountExport<unknown>): string | null {
|
|
132
|
+
const files = new Map<string, string>();
|
|
133
|
+
for (const [table, fate] of Object.entries(entry.tables)) {
|
|
134
|
+
if (!IDENTIFIER.test(table)) return `table « ${table} »: not an identifier`;
|
|
135
|
+
if (fate === 'custom') continue;
|
|
136
|
+
if ('skip' in fate) {
|
|
137
|
+
if (!fate.skip.trim()) return `table « ${table} »: a skip says why`;
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
if (!/^[a-z0-9][a-z0-9-]*\.json$/.test(fate.file)) {
|
|
141
|
+
return `table « ${table} »: file « ${fate.file} »`;
|
|
142
|
+
}
|
|
143
|
+
const scoped = `${fate.scope ?? 'workspace'}:${fate.file}`;
|
|
144
|
+
const other = files.get(scoped);
|
|
145
|
+
if (other) return `tables « ${other} » and « ${table} » write the same file`;
|
|
146
|
+
files.set(scoped, table);
|
|
147
|
+
if ((fate.where.match(/\?/g) ?? []).length !== 1) {
|
|
148
|
+
return `table « ${table} »: where takes exactly one ?`;
|
|
149
|
+
}
|
|
150
|
+
if (fate.where.includes(';')) return `table « ${table} »: where holds a ;`;
|
|
151
|
+
const columns = [
|
|
152
|
+
...fate.key,
|
|
153
|
+
...(fate.sealed ?? []),
|
|
154
|
+
...(fate.json ?? []),
|
|
155
|
+
...Object.keys(fate.dates ?? {}),
|
|
156
|
+
...(fate.omit ?? []),
|
|
157
|
+
...(fate.keep ?? [])
|
|
158
|
+
];
|
|
159
|
+
const bad = columns.find((column) => !IDENTIFIER.test(column));
|
|
160
|
+
if (bad) return `table « ${table} »: column « ${bad} »`;
|
|
161
|
+
const omitted = new Set(fate.omit ?? []);
|
|
162
|
+
const clash = (fate.sealed ?? []).find((column) => omitted.has(column));
|
|
163
|
+
if (clash) return `table « ${table} »: « ${clash} » is both sealed and omitted`;
|
|
164
|
+
}
|
|
165
|
+
if (entry.store && entry.store.omit.some((key) => !key.trim())) {
|
|
166
|
+
return 'store.omit holds an empty key';
|
|
167
|
+
}
|
|
168
|
+
return null;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
const MIN_PAGE = 10;
|
|
172
|
+
const MAX_PAGE = 1000;
|
|
173
|
+
/** A page aims at this many bytes: a table of data-URL images must not load a thousand of them at once. */
|
|
174
|
+
const PAGE_BYTES = 4 * 1024 * 1024;
|
|
175
|
+
|
|
176
|
+
/** A cell as JSON can carry it: dates to ISO 8601, large integers to strings, bytes to base64. */
|
|
177
|
+
function plainCell(value: unknown): unknown {
|
|
178
|
+
if (typeof value === 'bigint') {
|
|
179
|
+
return Number.isSafeInteger(Number(value)) ? Number(value) : value.toString();
|
|
180
|
+
}
|
|
181
|
+
if (value instanceof Date) return value.toISOString();
|
|
182
|
+
if (value instanceof Uint8Array) return { base64: Buffer.from(value).toString('base64') };
|
|
183
|
+
return value;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function isoOf(value: unknown, unit: 's' | 'ms'): unknown {
|
|
187
|
+
if (value === null || value === undefined || value === '') return null;
|
|
188
|
+
const n = Number(value);
|
|
189
|
+
if (!Number.isFinite(n) || n === 0) return value;
|
|
190
|
+
return new Date(unit === 's' ? n * 1000 : n).toISOString();
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* The rows of one declared table, oldest key first, one page at a time: the
|
|
195
|
+
* page size follows the bytes read, so a table of large cells never loads
|
|
196
|
+
* whole. Omitted columns never leave this function; sealed ones are opened by
|
|
197
|
+
* `open`, a `null` there meaning no key opens the cell (`onUnreadable` counts
|
|
198
|
+
* it).
|
|
199
|
+
*/
|
|
200
|
+
export async function* exportTableRows(
|
|
201
|
+
q: SdkQueryable,
|
|
202
|
+
table: string,
|
|
203
|
+
spec: SdkExportTable,
|
|
204
|
+
scopeId: number,
|
|
205
|
+
open: (blob: string) => Promise<string | null>,
|
|
206
|
+
onUnreadable: () => void = () => undefined
|
|
207
|
+
): AsyncGenerator<Record<string, unknown>> {
|
|
208
|
+
const omitted = new Set(spec.omit ?? []);
|
|
209
|
+
const sealed = new Set(spec.sealed ?? []);
|
|
210
|
+
const json = new Set(spec.json ?? []);
|
|
211
|
+
const dates = spec.dates ?? {};
|
|
212
|
+
const order = spec.key.map((column) => `\`${column}\``).join(', ');
|
|
213
|
+
let size = 200;
|
|
214
|
+
let last: unknown[] | null = null;
|
|
215
|
+
for (;;) {
|
|
216
|
+
const after: string =
|
|
217
|
+
last === null ? '' : ` AND (${order}) > (${spec.key.map(() => '?').join(', ')})`;
|
|
218
|
+
const rows: Record<string, unknown>[] = await q.query<Record<string, unknown>>(
|
|
219
|
+
`SELECT * FROM \`${table}\` WHERE (${spec.where})${after} ORDER BY ${order} LIMIT ${size}`,
|
|
220
|
+
[scopeId, ...(last ?? [])]
|
|
221
|
+
);
|
|
222
|
+
let bytes = 0;
|
|
223
|
+
for (const row of rows) {
|
|
224
|
+
const out: Record<string, unknown> = {};
|
|
225
|
+
for (const [column, raw] of Object.entries(row)) {
|
|
226
|
+
if (omitted.has(column)) continue;
|
|
227
|
+
let value: unknown = raw;
|
|
228
|
+
if (sealed.has(column) && typeof raw === 'string' && raw !== '') {
|
|
229
|
+
value = await open(raw);
|
|
230
|
+
if (value === null) onUnreadable();
|
|
231
|
+
}
|
|
232
|
+
if (json.has(column) && typeof value === 'string') {
|
|
233
|
+
try {
|
|
234
|
+
value = JSON.parse(value);
|
|
235
|
+
} catch {
|
|
236
|
+
// Un JSON qui ne se relit pas reste tel quel : la donnée prime sur sa forme.
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
if (column in dates) value = isoOf(value, dates[column]);
|
|
240
|
+
const plain =
|
|
241
|
+
sealed.has(column) && column.endsWith('_enc') ? column.slice(0, -4) : column;
|
|
242
|
+
out[plain !== column && !(plain in row) ? plain : column] = plainCell(value);
|
|
243
|
+
}
|
|
244
|
+
bytes += JSON.stringify(out).length;
|
|
245
|
+
yield out;
|
|
246
|
+
}
|
|
247
|
+
if (rows.length < size) return;
|
|
248
|
+
last = spec.key.map((column) => rows[rows.length - 1][column]);
|
|
249
|
+
size = Math.min(
|
|
250
|
+
MAX_PAGE,
|
|
251
|
+
Math.max(MIN_PAGE, Math.round((size * PAGE_BYTES) / Math.max(1, bytes)))
|
|
252
|
+
);
|
|
253
|
+
}
|
|
254
|
+
}
|