@rebasepro/server 0.25.1 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (179) hide show
  1. package/dist/{GCSStorageController-CRd9bFrc.js → GCSStorageController-BUClIfp0.js} +40 -3
  2. package/dist/GCSStorageController-BUClIfp0.js.map +1 -0
  3. package/dist/{S3StorageController-D00EFa5P.js → S3StorageController-CixCQEky.js} +58 -10
  4. package/dist/S3StorageController-CixCQEky.js.map +1 -0
  5. package/dist/access-BFAUvsSs.js +1897 -0
  6. package/dist/access-BFAUvsSs.js.map +1 -0
  7. package/dist/api/branch-routes.d.ts +25 -0
  8. package/dist/api/live-schema-routes.d.ts +10 -0
  9. package/dist/api/rest/api-generator.d.ts +9 -1
  10. package/dist/api/rest/idempotency.d.ts +15 -1
  11. package/dist/api/root-error-handler.d.ts +6 -5
  12. package/dist/api/types.d.ts +12 -0
  13. package/dist/{ast-schema-editor-mCeSC03x.js → ast-schema-editor-madTa9Hh.js} +2 -2
  14. package/dist/{ast-schema-editor-mCeSC03x.js.map → ast-schema-editor-madTa9Hh.js.map} +1 -1
  15. package/dist/auth/access.d.ts +10 -0
  16. package/dist/auth/adapter-middleware.d.ts +2 -1
  17. package/dist/auth/adapter-service-key.d.ts +24 -0
  18. package/dist/auth/admin-users-route.d.ts +3 -0
  19. package/dist/auth/api-keys/api-key-middleware.d.ts +21 -1
  20. package/dist/auth/api-keys/api-key-store.d.ts +8 -1
  21. package/dist/auth/api-keys/key-grant.d.ts +18 -1
  22. package/dist/auth/apple-oauth.d.ts +39 -10
  23. package/dist/auth/capabilities.d.ts +2 -0
  24. package/dist/auth/expired-token-sweep.d.ts +5 -0
  25. package/dist/auth/impersonation.d.ts +3 -1
  26. package/dist/auth/interfaces.d.ts +55 -0
  27. package/dist/auth/jwt.d.ts +8 -2
  28. package/dist/auth/oauth-signin-policy.d.ts +21 -35
  29. package/dist/auth/rls-scope.d.ts +17 -7
  30. package/dist/auth/rls-simulator-routes.d.ts +51 -0
  31. package/dist/auth/simulation-read-only.d.ts +28 -0
  32. package/dist/auth/simulation.d.ts +141 -0
  33. package/dist/auth/sql-rate-limit-store.d.ts +15 -1
  34. package/dist/{auth-Dkvo2tTJ.js → auth-DDXPw0mn.js} +527 -1883
  35. package/dist/auth-DDXPw0mn.js.map +1 -0
  36. package/dist/auth_adapter-B7RUMk6e.js +89 -0
  37. package/dist/auth_adapter-B7RUMk6e.js.map +1 -0
  38. package/dist/backend/create-backend.d.ts +6 -0
  39. package/dist/backend/options.d.ts +38 -5
  40. package/dist/{backend-DTAOsLQc.js → backend-CaG_Jyso.js} +9 -2
  41. package/dist/backend-CaG_Jyso.js.map +1 -0
  42. package/dist/{backup-B9m3cb3X.js → backup-B6brPSeB.js} +6 -6
  43. package/dist/{backup-B9m3cb3X.js.map → backup-B6brPSeB.js.map} +1 -1
  44. package/dist/boot/boot.d.ts +2 -15
  45. package/dist/boot/ddl-bootstrap.d.ts +34 -0
  46. package/dist/boot/env.d.ts +12 -0
  47. package/dist/boot/fetch-bundle.d.ts +5 -1
  48. package/dist/boot/provision-only.d.ts +23 -0
  49. package/dist/boot/provision.d.ts +53 -2
  50. package/dist/boot/role.d.ts +2 -0
  51. package/dist/boot/schema-change-recorder.d.ts +45 -0
  52. package/dist/boot/security-headers.d.ts +22 -0
  53. package/dist/branch-routes-B2vvEx53.js +84 -0
  54. package/dist/branch-routes-B2vvEx53.js.map +1 -0
  55. package/dist/{contract-routes-CsFXW7j4.js → contract-routes-CU2A3JnF.js} +2 -2
  56. package/dist/{contract-routes-CsFXW7j4.js.map → contract-routes-CU2A3JnF.js.map} +1 -1
  57. package/dist/cron/cron-loader.d.ts +4 -0
  58. package/dist/cron/cron-store.d.ts +8 -1
  59. package/dist/{cron-loader-Dj6lA4HQ.js → cron-loader-CC1Jy_KN.js} +14 -6
  60. package/dist/cron-loader-CC1Jy_KN.js.map +1 -0
  61. package/dist/{cron-routes-CwbLZguO.js → cron-routes-BK60Rn1r.js} +3 -3
  62. package/dist/{cron-routes-CwbLZguO.js.map → cron-routes-BK60Rn1r.js.map} +1 -1
  63. package/dist/{cron-scheduler-B04ACoUE.js → cron-scheduler-DyCjPlQR.js} +40 -27
  64. package/dist/cron-scheduler-DyCjPlQR.js.map +1 -0
  65. package/dist/{cron-store-DkyfuLOH.js → cron-store-DLn11M8H.js} +10 -9
  66. package/dist/cron-store-DLn11M8H.js.map +1 -0
  67. package/dist/{strings.es-BeZWAodn.js → data_driver-D0mg1Y_x.js} +2 -90
  68. package/dist/data_driver-D0mg1Y_x.js.map +1 -0
  69. package/dist/{ddl-bootstrap-Ck20GSIX.js → ddl-bootstrap-CjRrn12l.js} +91 -2
  70. package/dist/ddl-bootstrap-CjRrn12l.js.map +1 -0
  71. package/dist/deploy/pod-contract.d.ts +1 -1
  72. package/dist/{load-failure-L5hvfDoG.js → dynamic-import-Bi3XK5y-.js} +30 -18
  73. package/dist/dynamic-import-Bi3XK5y-.js.map +1 -0
  74. package/dist/email/dev-sink.d.ts +6 -0
  75. package/dist/email/index.d.ts +1 -1
  76. package/dist/{soft-delete-params-BmlVwecC.js → field-access-DYsvjI_P.js} +27 -57
  77. package/dist/field-access-DYsvjI_P.js.map +1 -0
  78. package/dist/{filter-dialect-O1HK6Nvr.js → filter-dialect-TrKFWLqT.js} +3 -3
  79. package/dist/{filter-dialect-O1HK6Nvr.js.map → filter-dialect-TrKFWLqT.js.map} +1 -1
  80. package/dist/{function-loader-KKDqTcIr.js → function-loader-eFigStYX.js} +34 -18
  81. package/dist/function-loader-eFigStYX.js.map +1 -0
  82. package/dist/functions/function-loader.d.ts +5 -0
  83. package/dist/functions/hono-like.d.ts +12 -0
  84. package/dist/functions/index.js +19 -1
  85. package/dist/functions/index.js.map +1 -1
  86. package/dist/functions/request-timeout.d.ts +3 -3
  87. package/dist/history/history-routes.d.ts +6 -0
  88. package/dist/history/retention-sweep.d.ts +5 -0
  89. package/dist/{history-recorder-B2rkuhuy.js → history-recorder-Bsau7h4X.js} +6 -6
  90. package/dist/history-recorder-Bsau7h4X.js.map +1 -0
  91. package/dist/{history-store-DPofGVDp.js → history-store-BQz5YCkW.js} +26 -2
  92. package/dist/history-store-BQz5YCkW.js.map +1 -0
  93. package/dist/index.d.ts +4 -2
  94. package/dist/index.es.js +3026 -611
  95. package/dist/index.es.js.map +1 -1
  96. package/dist/init/docs.d.ts +1 -1
  97. package/dist/init/health.d.ts +13 -0
  98. package/dist/init/process-safety.d.ts +4 -3
  99. package/dist/init/shutdown.d.ts +8 -0
  100. package/dist/init.d.ts +6 -1
  101. package/dist/jobs/job-store.d.ts +8 -1
  102. package/dist/{jobs-h1WRq3cp.js → jobs-Co6Ni5HP.js} +6 -6
  103. package/dist/jobs-Co6Ni5HP.js.map +1 -0
  104. package/dist/{jwt-D3oFauZI.js → jwt-rDS0884_.js} +4 -3
  105. package/dist/{jwt-D3oFauZI.js.map → jwt-rDS0884_.js.map} +1 -1
  106. package/dist/{keys-COzQ2aE8.js → keys-BXcdRNhJ.js} +2 -2
  107. package/dist/{keys-COzQ2aE8.js.map → keys-BXcdRNhJ.js.map} +1 -1
  108. package/dist/mcp/consent-page.d.ts +38 -1
  109. package/dist/mcp/mcp-routes.d.ts +9 -0
  110. package/dist/mcp/mcp-tools.d.ts +7 -1
  111. package/dist/mcp/oauth-store.d.ts +12 -5
  112. package/dist/metrics/history-recorder.d.ts +4 -1
  113. package/dist/metrics/history-store.d.ts +8 -0
  114. package/dist/metrics/index.d.ts +8 -0
  115. package/dist/metrics/realtime.d.ts +25 -0
  116. package/dist/{offline-BrqCRgih-BtDB1ecy.js → offline-AvuEcg7u-DFHL98OT.js} +127 -40
  117. package/dist/offline-AvuEcg7u-DFHL98OT.js.map +1 -0
  118. package/dist/{offline-store-BhYXViha-BW5j1We_.js → offline-store-BhYXViha-YTmjBpo2.js} +5 -5
  119. package/dist/{offline-store-BhYXViha-BW5j1We_.js.map → offline-store-BhYXViha-YTmjBpo2.js.map} +1 -1
  120. package/dist/{openapi-generator-BjHr6upl.js → openapi-generator-CyxEtYEX.js} +61 -22
  121. package/dist/openapi-generator-CyxEtYEX.js.map +1 -0
  122. package/dist/{proxy-CH8Jsovg.js → proxy-BYir8tWK.js} +4 -6
  123. package/dist/proxy-BYir8tWK.js.map +1 -0
  124. package/dist/{query-parser-otFZqJsC.js → query-parser-CDXWf2hH.js} +6 -5
  125. package/dist/{query-parser-otFZqJsC.js.map → query-parser-CDXWf2hH.js.map} +1 -1
  126. package/dist/{request-timeout-59crS-9u.js → request-timeout-DVjo-c_8.js} +4 -4
  127. package/dist/request-timeout-DVjo-c_8.js.map +1 -0
  128. package/dist/rls-simulator-routes-bEY7-5-u.js +190 -0
  129. package/dist/rls-simulator-routes-bEY7-5-u.js.map +1 -0
  130. package/dist/schema-edit/deployed-schema-guard.d.ts +47 -0
  131. package/dist/{schema-editor-routes-Bsut0bCC.js → schema-editor-routes-Dc3DU_jm.js} +3 -3
  132. package/dist/{schema-editor-routes-Bsut0bCC.js.map → schema-editor-routes-Dc3DU_jm.js.map} +1 -1
  133. package/dist/services/routed-realtime-service.d.ts +1 -1
  134. package/dist/soft-delete-params-BRONCOA2.js +59 -0
  135. package/dist/soft-delete-params-BRONCOA2.js.map +1 -0
  136. package/dist/{sql-export-routes-DmB_uZ4l.js → sql-export-routes-QycX8W8S.js} +38 -4
  137. package/dist/sql-export-routes-QycX8W8S.js.map +1 -0
  138. package/dist/{src-D5Y3N-S4.js → src-Bh9LRPb9.js} +21 -2
  139. package/dist/{src-D5Y3N-S4.js.map → src-Bh9LRPb9.js.map} +1 -1
  140. package/dist/storage/GCSStorageController.d.ts +11 -1
  141. package/dist/storage/LocalStorageController.d.ts +16 -4
  142. package/dist/storage/S3StorageController.d.ts +6 -2
  143. package/dist/storage/index.d.ts +1 -0
  144. package/dist/storage/maintenance-keys.d.ts +17 -0
  145. package/dist/storage/maintenance-routes.d.ts +28 -5
  146. package/dist/storage/maintenance.d.ts +136 -4
  147. package/dist/storage/orphan-scans.d.ts +120 -0
  148. package/dist/storage/policies.d.ts +1 -1
  149. package/dist/storage/streams.d.ts +17 -0
  150. package/dist/storage/types.d.ts +26 -33
  151. package/dist/strings.es-B7QLnHh2.js +96 -0
  152. package/dist/strings.es-B7QLnHh2.js.map +1 -0
  153. package/dist/{upload-validation-Bgo626uY.js → upload-validation-CkkQGs-j.js} +40 -2
  154. package/dist/upload-validation-CkkQGs-j.js.map +1 -0
  155. package/dist/utils/load-failure.d.ts +9 -0
  156. package/package.json +5 -5
  157. package/dist/GCSStorageController-CRd9bFrc.js.map +0 -1
  158. package/dist/S3StorageController-D00EFa5P.js.map +0 -1
  159. package/dist/auth-Dkvo2tTJ.js.map +0 -1
  160. package/dist/auth_adapter-CIItCTc8.js +0 -21
  161. package/dist/auth_adapter-CIItCTc8.js.map +0 -1
  162. package/dist/backend-DTAOsLQc.js.map +0 -1
  163. package/dist/cron-loader-Dj6lA4HQ.js.map +0 -1
  164. package/dist/cron-scheduler-B04ACoUE.js.map +0 -1
  165. package/dist/cron-store-DkyfuLOH.js.map +0 -1
  166. package/dist/ddl-bootstrap-Ck20GSIX.js.map +0 -1
  167. package/dist/function-loader-KKDqTcIr.js.map +0 -1
  168. package/dist/history-recorder-B2rkuhuy.js.map +0 -1
  169. package/dist/history-store-DPofGVDp.js.map +0 -1
  170. package/dist/jobs-h1WRq3cp.js.map +0 -1
  171. package/dist/load-failure-L5hvfDoG.js.map +0 -1
  172. package/dist/offline-BrqCRgih-BtDB1ecy.js.map +0 -1
  173. package/dist/openapi-generator-BjHr6upl.js.map +0 -1
  174. package/dist/proxy-CH8Jsovg.js.map +0 -1
  175. package/dist/request-timeout-59crS-9u.js.map +0 -1
  176. package/dist/soft-delete-params-BmlVwecC.js.map +0 -1
  177. package/dist/sql-export-routes-DmB_uZ4l.js.map +0 -1
  178. package/dist/strings.es-BeZWAodn.js.map +0 -1
  179. package/dist/upload-validation-Bgo626uY.js.map +0 -1
@@ -5,7 +5,7 @@
5
5
  * optional peer dependency — users who don't need GCS never pay for the
6
6
  * import cost or need it installed.
7
7
  */
8
- import { StorageController, GCSStorageConfig, type StoredObjectInfo, type StoredObjectRange, type PutObjectOptions, type StoredObjectStream } from "./types.js";
8
+ import { StorageController, GCSStorageConfig, type StoredObjectInfo, type StoredObjectRange, type PutObjectOptions, type PutObjectStreamProps, type PutObjectStreamResult, type StoredObjectStream } from "./types.js";
9
9
  import { UploadFileProps, UploadFileResult, DownloadConfig, DownloadMetadata, StorageListResult } from "@rebasepro/types";
10
10
  /**
11
11
  * Google Cloud Storage implementation of `StorageController`.
@@ -24,7 +24,17 @@ export declare class GCSStorageController implements StorageController {
24
24
  * @see S3StorageController.knownBuckets
25
25
  */
26
26
  knownBuckets(): string[];
27
+ /** See {@link StorageController.storageLocation}: bucket names are global. */
28
+ storageLocation(): string;
27
29
  putObject({ file, key, metadata, bucket }: UploadFileProps, options?: PutObjectOptions): Promise<UploadFileResult>;
30
+ /**
31
+ * See {@link StorageController.putObjectStream}.
32
+ *
33
+ * `createWriteStream`, not the `save` above: `save` retries by opening its
34
+ * source again, and this source is a stream with nothing left to give, so a
35
+ * retry would finish an object holding only what came after the failure.
36
+ */
37
+ putObjectStream({ stream, key, contentType, bucket, maxBytes }: PutObjectStreamProps): Promise<PutObjectStreamResult>;
28
38
  /**
29
39
  * Read the object's metadata and nothing else — no signing.
30
40
  *
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * Local filesystem storage controller
3
3
  */
4
- import { StorageController, LocalStorageConfig, type StoredObjectInfo, type StoredObjectRange, type PutObjectOptions, type StoredObjectStream } from "./types.js";
4
+ import { StorageController, LocalStorageConfig, type StoredObjectInfo, type StoredObjectRange, type PutObjectOptions, type PutObjectStreamProps, type PutObjectStreamResult, type StoredObjectStream } from "./types.js";
5
5
  import { UploadFileProps, UploadFileResult, DownloadConfig, StorageListResult } from "@rebasepro/types";
6
6
  /**
7
7
  * Bucket used when a call names none.
@@ -67,6 +67,11 @@ export declare class LocalStorageController implements StorageController {
67
67
  * not have been written to yet.
68
68
  */
69
69
  knownBuckets(): string[];
70
+ /**
71
+ * See {@link StorageController.storageLocation}: the default bucket's
72
+ * directory, with the storage root's links resolved.
73
+ */
74
+ storageLocation(): string;
70
75
  /**
71
76
  * Ensure directory exists, creating it if necessary
72
77
  */
@@ -104,6 +109,13 @@ export declare class LocalStorageController implements StorageController {
104
109
  */
105
110
  private validateFile;
106
111
  putObject({ file, key, metadata, bucket }: UploadFileProps, options?: PutObjectOptions): Promise<UploadFileResult>;
112
+ /** See {@link StorageController.putObjectStream}. */
113
+ putObjectStream({ stream, key, contentType, bucket, maxBytes }: PutObjectStreamProps): Promise<PutObjectStreamResult>;
114
+ /**
115
+ * Write an object and its metadata sidecar. `metadata` is a function when
116
+ * the size is only known once the bytes have been written.
117
+ */
118
+ private writeObject;
107
119
  getSignedUrl(key: string, bucket?: string): Promise<DownloadConfig>;
108
120
  getObject(key: string, bucket?: string): Promise<File | null>;
109
121
  /** The file a read of `key` addresses — `getObject`'s resolution, `local://` URLs included. */
@@ -128,11 +140,11 @@ export declare class LocalStorageController implements StorageController {
128
140
  /**
129
141
  * A folder's entries for a listing page.
130
142
  *
131
- * A page token is an index into these entries, and every continuation
132
- * page used to read the whole folder again to find its place: a folder of
143
+ * A page token names the last of these entries a page read, and every
144
+ * continuation page used to read the whole folder again to find its place: a folder of
133
145
  * n entries listed in pages of p cost n/p full reads. A first page reads
134
146
  * the folder and, when there is a next page, keeps the read; a
135
- * continuation page indexes into it while all of these hold:
147
+ * continuation page finds its place in it while all of these hold:
136
148
  *
137
149
  * - the folder's mtime is the one read with it. Adding, removing or
138
150
  * renaming an entry changes it, which is the invalidation;
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * S3-compatible storage controller (works with AWS S3 and MinIO)
3
3
  */
4
- import { S3StorageConfig, StorageController, type StoredObjectInfo, type StoredObjectRange, type PutObjectOptions, type StoredObjectStream } from "./types.js";
4
+ import { S3StorageConfig, StorageController, type StoredObjectInfo, type StoredObjectRange, type PutObjectOptions, type PutObjectStreamProps, type PutObjectStreamResult, type StoredObjectStream } from "./types.js";
5
5
  import { DownloadConfig, DownloadMetadata, StorageListResult, UploadFileProps, UploadFileResult } from "@rebasepro/types";
6
6
  /**
7
7
  * S3-compatible storage implementation
@@ -24,6 +24,8 @@ export declare class S3StorageController implements StorageController {
24
24
  * can reach — and a mistyped one came back as "file not found".
25
25
  */
26
26
  knownBuckets(): string[];
27
+ /** See {@link StorageController.storageLocation}: the endpoint (AWS when none) and the bucket. */
28
+ storageLocation(): string;
27
29
  /** See {@link StorageController.maxFileSize}. */
28
30
  maxFileSize(): number;
29
31
  /**
@@ -35,8 +37,10 @@ export declare class S3StorageController implements StorageController {
35
37
  */
36
38
  private getBucket;
37
39
  putObject({ file, key, metadata, bucket }: UploadFileProps, options?: PutObjectOptions): Promise<UploadFileResult>;
40
+ /** See {@link StorageController.putObjectStream}. */
41
+ putObjectStream({ stream, key, contentType, bucket, maxBytes }: PutObjectStreamProps): Promise<PutObjectStreamResult>;
38
42
  /**
39
- * Upload a large file in parts, one part in memory at a time.
43
+ * Upload a large object in parts, one part in memory at a time.
40
44
  *
41
45
  * Reading the whole file into one buffer held it twice over (the
42
46
  * `arrayBuffer()`, then the SDK's view and the multipart parser's copy
@@ -15,6 +15,7 @@ export * from "./types.js";
15
15
  export { LocalStorageController } from "./LocalStorageController.js";
16
16
  export { S3StorageController } from "./S3StorageController.js";
17
17
  export { GCSStorageController } from "./GCSStorageController.js";
18
+ export { ObjectTooLargeError } from "./streams.js";
18
19
  export { createStorageRoutes } from "./routes.js";
19
20
  export type { StorageRoutesConfig } from "./routes.js";
20
21
  export * from "./storage-registry.js";
@@ -0,0 +1,17 @@
1
+ /**
2
+ * Where storage maintenance keeps its own records inside a source: the state
3
+ * and result of an orphan scan (`rebase storage orphans`), which a worker
4
+ * writes and any server process reads back.
5
+ *
6
+ * Reserved at the route layer like the rendition space, in both directions. A
7
+ * scan's result is the list a later delete acts on, so a caller who could
8
+ * write one could name files for an administrator's delete to remove; and it
9
+ * lists keys a caller may not be allowed to see.
10
+ */
11
+ export declare const MAINTENANCE_PREFIX = "_rebase/maintenance/";
12
+ /**
13
+ * True for a key in the reserved maintenance space — the prefix itself, or
14
+ * anything under it. Folded the way the filesystem under a local bucket folds
15
+ * it, as {@link isRenditionKey} is.
16
+ */
17
+ export declare function isMaintenanceKey(key: string): boolean;
@@ -1,23 +1,46 @@
1
1
  /**
2
2
  * `/admin/storage` — the routes behind `rebase storage orphans` and
3
3
  * `rebase storage move`. Admin-only; the caller mounts them behind the admin
4
- * gate (`database:read` to read, `database:write` to delete or move).
4
+ * gate (`database:read` to read, `database:write` for every other method).
5
5
  *
6
- * GET /orphans?storageId=&olderThanHours= objects no row names
7
- * DELETE /orphans?storageId=&olderThanHours= delete them, found again first
6
+ * POST /orphans/scans?storageId=&olderThanHours= start a scan, in the background
7
+ * GET /orphans/scans/:id?storageId= its progress, then its report
8
+ * DELETE /orphans?scan=:id&storageId= delete a completed scan's orphans, in the background
9
+ * GET /orphans?storageId=&olderThanHours= deprecated: a scan inside the request
10
+ * DELETE /orphans?storageId=&olderThanHours= deprecated: scan and delete inside the request
8
11
  * POST /move { from, to, prefix?, cursor?, limit?, deleteSource? }
12
+ * POST /move/verify (the same body) what a move left behind
9
13
  */
10
14
  import { Hono } from "hono";
11
- import type { CollectionConfig, DataDriver } from "@rebasepro/types";
15
+ import type { CollectionCallbacks, CollectionConfig, DataDriver } from "@rebasepro/types";
12
16
  import type { HonoEnv } from "../api/types.js";
17
+ import type { JobQueueClient } from "../jobs/types.js";
13
18
  import type { StorageController } from "./types.js";
14
19
  import type { StorageRegistry } from "./storage-registry.js";
20
+ import { type OrphanScanService } from "./orphan-scans.js";
15
21
  export interface StorageMaintenanceRoutesConfig {
16
22
  registry?: StorageRegistry;
17
23
  controller?: StorageController;
18
24
  /** The collections whose rows are read for references, read per request. */
19
25
  collections: () => readonly CollectionConfig[];
20
- /** The unscoped driver serving a collection: every row, whoever wrote it. */
26
+ /**
27
+ * The unscoped driver serving a collection: every row, whoever wrote it.
28
+ * Its `scanStoredRows` is read when it has one — rows as stored, past
29
+ * every read hook.
30
+ */
21
31
  driverFor: (collection: CollectionConfig) => DataDriver | undefined;
32
+ /** The global callbacks, read per request: a global read hook makes every collection hooked. */
33
+ globalCallbacks?: () => CollectionCallbacks | undefined;
34
+ /** Row history, when kept: a file only an entry names is restored by a revert. */
35
+ history?: {
36
+ scanStoredValues?(pageSize?: number): AsyncIterable<unknown[]>;
37
+ };
38
+ /** The durable job queue a scan runs on, read per scan. Absent: scans run in this process. */
39
+ jobs?: () => JobQueueClient | undefined;
40
+ /**
41
+ * The scan service, when the backend shares one with the job queue's
42
+ * `storage.orphans` handler. Built from the fields above when left out.
43
+ */
44
+ orphanScans?: OrphanScanService;
22
45
  }
23
46
  export declare function createStorageMaintenanceRoutes(config: StorageMaintenanceRoutesConfig): Hono<HonoEnv>;
@@ -11,7 +11,7 @@
11
11
  * whichever property holds it. A file is reported only when nothing points at
12
12
  * it at all.
13
13
  */
14
- import type { CollectionConfig, DataDriver } from "@rebasepro/types";
14
+ import type { CollectionCallbacks, CollectionConfig, DataDriver } from "@rebasepro/types";
15
15
  import type { StorageController } from "./types.js";
16
16
  /** One stored object, as a listing describes it. */
17
17
  export interface StoredObjectEntry {
@@ -51,6 +51,12 @@ export interface UnscannedCollection {
51
51
  collection: string;
52
52
  reason: string;
53
53
  }
54
+ /**
55
+ * Whether reading a collection through `fetchCollection` could hide a row or
56
+ * change a value: a `beforeQuery` or an `afterRead`, the collection's own, a
57
+ * global one, or a property's.
58
+ */
59
+ export declare function hasReadHooks(collection: CollectionConfig, globalCallbacks?: CollectionCallbacks): boolean;
54
60
  /**
55
61
  * Remove from `keys` every key some row names.
56
62
  *
@@ -59,13 +65,28 @@ export interface UnscannedCollection {
59
65
  * serves the collection. What is left in `keys` afterwards is named by no row.
60
66
  * A collection that could not be read is returned, and its rows count for
61
67
  * nothing: a caller must not treat what is left as unused when any is.
68
+ *
69
+ * Rows are read as stored, through the driver's `scanStoredRows`. A driver
70
+ * without it is read through `fetchCollection` only for a collection with no
71
+ * read hooks: a `beforeQuery` scoping rows to `context.user` — there is none
72
+ * here — or an `afterRead` rewriting a file field would hide a reference, and
73
+ * the file it names would be deleted as unused. Such a collection is
74
+ * returned as unscanned instead.
62
75
  */
63
76
  export declare function removeReferencedKeys(keys: Set<string>, collections: readonly CollectionConfig[], driverFor: (collection: CollectionConfig) => DataDriver | undefined, options?: {
64
77
  pageSize?: number;
78
+ globalCallbacks?: CollectionCallbacks;
79
+ /** Counted into as it goes, and `onProgress` called after each page. */
80
+ progress?: ScanProgress;
81
+ onProgress?: () => void;
82
+ /** Checked between pages: an aborted scan stops, and throws. */
83
+ signal?: AbortSignal;
65
84
  }): Promise<{
66
85
  scanned: string[];
67
86
  unscanned: UnscannedCollection[];
68
87
  }>;
88
+ /** How row history is named among the scanned and unscanned collections. */
89
+ export declare const HISTORY_SCAN_NAME = "(row history)";
69
90
  /** What `findOrphans` found. */
70
91
  export interface OrphanReport {
71
92
  /** Objects in the source, internal ones aside. */
@@ -80,6 +101,42 @@ export interface OrphanReport {
80
101
  scanned: string[];
81
102
  unscanned: UnscannedCollection[];
82
103
  }
104
+ /** Row history, as a scan reads it. */
105
+ export interface StoredHistory {
106
+ scanStoredValues?(pageSize?: number): AsyncIterable<unknown[]>;
107
+ }
108
+ /** Where references to stored files are read from: every row, and history. */
109
+ export interface ReferenceSources {
110
+ collections: readonly CollectionConfig[];
111
+ driverFor: (collection: CollectionConfig) => DataDriver | undefined;
112
+ /** The callbacks every collection runs, which may scope or rewrite its reads. */
113
+ globalCallbacks?: CollectionCallbacks;
114
+ /**
115
+ * Row history, when the backend keeps it: a revert restores the values an
116
+ * entry holds, so a file only history names is still in use. One that
117
+ * cannot be read is reported unscanned, as {@link HISTORY_SCAN_NAME}.
118
+ */
119
+ history?: StoredHistory;
120
+ }
121
+ /** How far a scan has got. Counted into in place, so a caller can report it while it runs. */
122
+ export interface ScanProgress {
123
+ phase: "listing" | "rows" | "history" | "deleting" | "done";
124
+ /** Objects listed in the source so far. */
125
+ filesScanned: number;
126
+ /** Rows read so far, over every collection. */
127
+ rowsScanned: number;
128
+ /** History entries read so far. */
129
+ historyEntriesScanned: number;
130
+ /** Collections finished, read or not. */
131
+ collectionsScanned: number;
132
+ collectionsTotal: number;
133
+ /** The collection being read now. */
134
+ currentCollection?: string;
135
+ /** The collections that could not be read, as they are found. */
136
+ unscanned: UnscannedCollection[];
137
+ }
138
+ /** A progress record at the start of a scan. */
139
+ export declare function emptyScanProgress(): ScanProgress;
83
140
  /**
84
141
  * The objects of one source that no row names.
85
142
  *
@@ -87,13 +144,52 @@ export interface OrphanReport {
87
144
  * an orphan: the admin panel uploads a file before the row that names it
88
145
  * is saved. One whose age the listing does not say is not either.
89
146
  */
90
- export declare function findOrphans(options: {
147
+ export declare function findOrphans(options: ReferenceSources & {
91
148
  controller: StorageController;
92
- collections: readonly CollectionConfig[];
93
- driverFor: (collection: CollectionConfig) => DataDriver | undefined;
94
149
  olderThanMs: number;
95
150
  now?: number;
151
+ /** Counted into as the scan goes; `onProgress` is called after each page. */
152
+ progress?: ScanProgress;
153
+ onProgress?: () => void;
154
+ /** Checked between pages: an aborted scan stops, and throws. */
155
+ signal?: AbortSignal;
96
156
  }): Promise<OrphanReport>;
157
+ /** What deleting a scan's orphans did. */
158
+ export interface OrphanDeletion {
159
+ deleted: string[];
160
+ /** Candidates not deleted, because something changed since the scan. */
161
+ kept: {
162
+ key: string;
163
+ reason: string;
164
+ }[];
165
+ failed: {
166
+ key: string;
167
+ error: string;
168
+ }[];
169
+ /** Collections the re-check could not read; when any is, nothing was deleted. */
170
+ unscanned: UnscannedCollection[];
171
+ }
172
+ /**
173
+ * Delete the orphans an earlier scan found — each only if it is still one.
174
+ *
175
+ * Every row and history entry is read again, against the candidates alone
176
+ * (the source is not listed again): a row written since the scan may name one
177
+ * now. When a collection cannot be read, nothing is deleted. Then, right
178
+ * before each delete, the object is asked for again: one that is gone, was
179
+ * written again since the scan (another size or time), or is now inside the
180
+ * grace period is kept.
181
+ */
182
+ export declare function deleteScannedOrphans(options: ReferenceSources & {
183
+ controller: StorageController;
184
+ candidates: readonly StoredObjectEntry[];
185
+ olderThanMs: number;
186
+ now?: () => number;
187
+ progress?: ScanProgress;
188
+ onProgress?: () => void;
189
+ /** Called after each file deleted. */
190
+ onDeleted?: (key: string) => void;
191
+ signal?: AbortSignal;
192
+ }): Promise<OrphanDeletion>;
97
193
  /** What happened to one object in a move. */
98
194
  export interface MovedObject {
99
195
  key: string;
@@ -124,3 +220,39 @@ export declare function moveObjects(options: {
124
220
  moved: MovedObject[];
125
221
  cursor?: string;
126
222
  }>;
223
+ /** A file a move did not move, and why it counts as not moved. */
224
+ export interface NotMovedObject {
225
+ key: string;
226
+ reason: string;
227
+ }
228
+ /**
229
+ * List the source again after a move and name every file it did not move,
230
+ * `limit` source objects per call; hand the returned `cursor` back for the
231
+ * next ones.
232
+ *
233
+ * Defence in depth: a move reports what its own walk saw, and a walk that
234
+ * skipped files (as the local listing's page token once did, over a folder
235
+ * the move was emptying) reported success over the half it moved. A fresh
236
+ * listing does not share the walk's state. With `deleteSource`, every file
237
+ * still in the source is not moved; without it, a file the target holds no
238
+ * copy of the same size of.
239
+ */
240
+ export declare function verifyMove(options: {
241
+ from: StorageController;
242
+ to: StorageController;
243
+ prefix?: string;
244
+ cursor?: string;
245
+ limit?: number;
246
+ deleteSource?: boolean;
247
+ }): Promise<{
248
+ checked: number;
249
+ notMoved: NotMovedObject[];
250
+ cursor?: string;
251
+ }>;
252
+ /**
253
+ * Whether two sources store in the same place — one bucket or directory under
254
+ * two names. Asked by their `storageLocation` when both can say; otherwise by
255
+ * a probe object written to `a` under the internal folder, looked for in `b`,
256
+ * and deleted.
257
+ */
258
+ export declare function storesInSamePlace(a: StorageController, b: StorageController): Promise<boolean>;
@@ -0,0 +1,120 @@
1
+ import type { CollectionCallbacks, CollectionConfig, DataDriver } from "@rebasepro/types";
2
+ import type { JobQueueClient } from "../jobs/types.js";
3
+ import type { StorageController } from "./types.js";
4
+ import type { StorageRegistry } from "./storage-registry.js";
5
+ import { type OrphanReport, type ScanProgress, type StoredHistory } from "./maintenance.js";
6
+ /** The job task a scan, or a scan's delete, runs as. Frozen: workers of another release claim it by name. */
7
+ export declare const STORAGE_ORPHANS_TASK = "storage.orphans";
8
+ /** Where scan records are kept inside the scanned source. */
9
+ export declare const ORPHAN_SCAN_FOLDER = "_rebase/maintenance/orphan-scans";
10
+ /**
11
+ * How long after it completed a scan's result may still be deleted from.
12
+ * Every candidate is checked again before it goes, so this is not what keeps
13
+ * a file in use; it keeps a delete acting on what the person just read.
14
+ */
15
+ export declare const ORPHAN_SCAN_TTL_MS: number;
16
+ /**
17
+ * A scan, or a delete, that has written no progress for this long is reported
18
+ * failed: the process running it stopped — a deploy, a crash — and nothing
19
+ * else will finish it. Progress is written at least once a page.
20
+ */
21
+ export declare const ORPHAN_SCAN_STALL_MS: number;
22
+ /** The default grace period: an upload whose row is not saved yet is a day old at most. */
23
+ export declare const DEFAULT_OLDER_THAN_HOURS = 24;
24
+ /** Who runs a scan: the job queue's workers, or the server process that was asked. */
25
+ export type OrphanScanRunner = "worker" | "in-process";
26
+ export type OrphanScanStatus = "queued" | "running" | "completed" | "failed";
27
+ /** A delete from a completed scan, as its record holds it. */
28
+ export interface OrphanScanDeletion {
29
+ status: OrphanScanStatus;
30
+ runner: OrphanScanRunner;
31
+ jobId?: string;
32
+ requestedAt: string;
33
+ startedAt?: string;
34
+ updatedAt: string;
35
+ completedAt?: string;
36
+ progress: ScanProgress & {
37
+ candidates: number;
38
+ deleted: number;
39
+ };
40
+ result?: {
41
+ deleted: string[];
42
+ kept: {
43
+ key: string;
44
+ reason: string;
45
+ }[];
46
+ failed: {
47
+ key: string;
48
+ error: string;
49
+ }[];
50
+ };
51
+ error?: string;
52
+ /** Set on the way out when a running delete has stopped reporting. */
53
+ note?: string;
54
+ }
55
+ /** A scan's record: what `GET /orphans/scans/:id` answers. */
56
+ export interface OrphanScanState {
57
+ id: string;
58
+ storageId: string;
59
+ olderThanHours: number;
60
+ runner: OrphanScanRunner;
61
+ jobId?: string;
62
+ status: OrphanScanStatus;
63
+ createdAt: string;
64
+ startedAt?: string;
65
+ updatedAt: string;
66
+ completedAt?: string;
67
+ progress: ScanProgress;
68
+ /** The report, once completed: what the synchronous `GET /orphans` answered. */
69
+ result?: OrphanReport;
70
+ error?: string;
71
+ /** Said on the way out, never stored: why a queued scan may not have started, or that one stalled. */
72
+ note?: string;
73
+ deletion?: OrphanScanDeletion;
74
+ }
75
+ /** A `storage.orphans` job's payload. */
76
+ export interface OrphanScanTaskPayload {
77
+ scanId: string;
78
+ storageId: string;
79
+ mode: "scan" | "delete";
80
+ }
81
+ export interface OrphanScanServiceConfig {
82
+ registry?: StorageRegistry;
83
+ controller?: StorageController;
84
+ /** The collections whose rows are read for references, read when a scan runs. */
85
+ collections: () => readonly CollectionConfig[];
86
+ /** The unscoped driver serving a collection. See `StorageMaintenanceRoutesConfig.driverFor`. */
87
+ driverFor: (collection: CollectionConfig) => DataDriver | undefined;
88
+ globalCallbacks?: () => CollectionCallbacks | undefined;
89
+ history?: StoredHistory;
90
+ /**
91
+ * The durable job queue, read when a scan starts: present, the scan is a
92
+ * `storage.orphans` job its workers run; absent, it runs in this process.
93
+ */
94
+ jobs?: () => JobQueueClient | undefined;
95
+ /** For tests. */
96
+ now?: () => number;
97
+ }
98
+ export interface OrphanScanService {
99
+ /** Record a scan and set it going. Resolves with its record, `queued`. */
100
+ start(options: {
101
+ storageId?: string;
102
+ olderThanHours?: number;
103
+ }): Promise<OrphanScanState>;
104
+ /** A scan's record, or `undefined` when this source holds none by that id. */
105
+ status(storageId: string | undefined, id: string): Promise<OrphanScanState | undefined>;
106
+ /** Start deleting a completed scan's orphans. Throws the refusal when it may not. */
107
+ requestDeletion(storageId: string | undefined, id: string): Promise<OrphanScanState>;
108
+ /** Run a `storage.orphans` job: the scan, or its delete. */
109
+ runTask(payload: OrphanScanTaskPayload, signal?: AbortSignal): Promise<void>;
110
+ }
111
+ /**
112
+ * The storage source a request names, or the 404 that lists the ones served.
113
+ */
114
+ export declare function resolveStorageSource(sources: {
115
+ registry?: StorageRegistry;
116
+ controller?: StorageController;
117
+ }, rawId: unknown, label: string): StorageController;
118
+ /** `olderThanHours` as a request sends it, checked. */
119
+ export declare function parseOlderThanHours(raw: string | number | undefined): number;
120
+ export declare function createOrphanScanService(config: OrphanScanServiceConfig): OrphanScanService;
@@ -31,7 +31,7 @@
31
31
  * Ownership frequently lives in a row rather than in a key, which no pattern can
32
32
  * express. So when policies match nothing, an explicit `storageAuthorize` is
33
33
  * consulted before the request is refused. Present, it can widen; absent, the
34
- * answer is no. Policies never *narrow* what the hook allowed and the hook never
34
+ * answer is no. A policy that matched and did not grant is a denial, not silence. Policies never *narrow* what the hook allowed and the hook never
35
35
  * overrides a policy denial — each can only say yes, which keeps "who allowed
36
36
  * this?" answerable by reading either one alone.
37
37
  */
@@ -13,6 +13,23 @@ type ByteSource = AsyncIterable<Uint8Array>;
13
13
  * socket.
14
14
  */
15
15
  export declare function webStreamFrom(source: ByteSource | (() => ByteSource)): ReadableStream<Uint8Array>;
16
+ /** A streamed write ran past the limit its caller set. See {@link meteredChunks}. */
17
+ export declare class ObjectTooLargeError extends Error {
18
+ readonly maxBytes: number;
19
+ constructor(maxBytes: number);
20
+ }
21
+ /**
22
+ * A stream's chunks, counted as they pass, stopping at `maxBytes`.
23
+ *
24
+ * Stopping cancels the stream, so a body being refused is not read to its end
25
+ * first. The error surfaces at the consumer's next read, which is how a write in
26
+ * progress learns to abandon itself rather than finishing an object it was
27
+ * never allowed to keep.
28
+ */
29
+ export declare function meteredChunks(stream: ReadableStream<Uint8Array>, maxBytes?: number): {
30
+ chunks: AsyncGenerator<Uint8Array>;
31
+ size: () => number;
32
+ };
16
33
  /**
17
34
  * A file on disk, or an inclusive byte range of it, as a web stream. The file
18
35
  * is opened on the first read; see {@link webStreamFrom}.
@@ -2,6 +2,7 @@
2
2
  * Storage configuration and types for Rebase backend
3
3
  */
4
4
  import { UploadFileProps, UploadFileResult, DownloadConfig, DownloadMetadata, StorageListResult } from "@rebasepro/types";
5
+ import type { PutObjectStreamProps, PutObjectStreamResult, StoredObjectInfo, StoredObjectRange, StoredObjectStream } from "@rebasepro/types";
5
6
  /**
6
7
  * Local filesystem storage configuration
7
8
  */
@@ -91,7 +92,7 @@ export interface GCSStorageConfig {
91
92
  */
92
93
  export type BackendStorageConfig = LocalStorageConfig | S3StorageConfig | GCSStorageConfig;
93
94
  /** What a storage request is trying to do to an object. */
94
- export type { StorageOperation, StorageAuthorizeUser, StorageAuthorizeContext, StorageAuthorizeData, StorageAuthorize } from "@rebasepro/types";
95
+ export type { StorageOperation, StorageAuthorizeUser, StorageAuthorizeContext, StorageAuthorizeData, StorageAuthorize, PutObjectStreamProps, PutObjectStreamResult, StoredObjectInfo, StoredObjectRange, StoredObjectStream } from "@rebasepro/types";
95
96
  /**
96
97
  * How the server writes an object. See {@link StorageController.putObject}.
97
98
  */
@@ -171,6 +172,19 @@ export interface StorageController {
171
172
  bucket?: string;
172
173
  range?: StoredObjectRange;
173
174
  }): Promise<StoredObjectStream | null>;
175
+ /**
176
+ * Write an object from a stream, a chunk at a time, for bytes the server
177
+ * has not got all of yet: a request body on its way through.
178
+ *
179
+ * Past `maxBytes` this rejects with {@link ObjectTooLargeError}, and on that
180
+ * or any other failure no object appears at the key. `maxFileSize` and
181
+ * `allowedMimeTypes` do not apply; `maxBytes` is the caller's own limit.
182
+ *
183
+ * The stream can be read only once, so unlike {@link putObject} a failed
184
+ * attempt cannot be retried here. A caller that must survive one keeps its
185
+ * own copy.
186
+ */
187
+ putObjectStream?(props: PutObjectStreamProps): Promise<PutObjectStreamResult>;
174
188
  /**
175
189
  * Delete an object
176
190
  */
@@ -210,6 +224,17 @@ export interface StorageController {
210
224
  * when the caller names none, and each provider maps it to its own.
211
225
  */
212
226
  knownBuckets?(): string[];
227
+ /**
228
+ * Where this controller's default bucket physically is, as a string two
229
+ * controllers return alike exactly when they store in the same place: a
230
+ * resolved directory, an endpoint and bucket.
231
+ *
232
+ * Two sources declared over one bucket or directory are one store under
233
+ * two names, and `rebase storage move --delete-source` between them found
234
+ * every object "already there" — it is the same object — and deleted it
235
+ * from both. A controller without this is found out by a probe object.
236
+ */
237
+ storageLocation?(): string;
213
238
  /**
214
239
  * The largest file this controller accepts, in bytes.
215
240
  *
@@ -222,38 +247,6 @@ export interface StorageController {
222
247
  */
223
248
  maxFileSize?(): number;
224
249
  }
225
- /**
226
- * What serving an object needs to know before deciding whether to read it.
227
- * See {@link StorageController.statObject}.
228
- */
229
- export interface StoredObjectInfo {
230
- /** The whole object's size in bytes. */
231
- size: number;
232
- /** The stored content type — the uploader's claim, not a sniffed one. */
233
- contentType: string;
234
- /**
235
- * The object's own modification time, in milliseconds since the epoch.
236
- * The routes build `ETag`, `Last-Modified` and the transform cache key from
237
- * it, so it must be the object's version, not the time of the read.
238
- */
239
- lastModified: number;
240
- }
241
- /** An inclusive byte range, as `Range: bytes=start-end` names one. */
242
- export interface StoredObjectRange {
243
- start: number;
244
- end: number;
245
- }
246
- /** An object's bytes as a stream. See {@link StorageController.getObjectStream}. */
247
- export interface StoredObjectStream extends StoredObjectInfo {
248
- /** The bytes: the range when {@link range} is set, the whole object otherwise. */
249
- stream: ReadableStream<Uint8Array>;
250
- /**
251
- * The range {@link stream} holds — the one asked for — or absent when it
252
- * holds the whole object. A controller may answer a range with the whole
253
- * object, as HTTP lets a server do; the caller then serves it whole.
254
- */
255
- range?: StoredObjectRange;
256
- }
257
250
  /**
258
251
  * Default maximum file size (50MB)
259
252
  */