@minca/core 0.1.7 → 0.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. package/dist/elements/auth-styles.d.ts +1 -1
  2. package/dist/elements/auth-styles.d.ts.map +1 -1
  3. package/dist/elements/auth-styles.js +16 -8
  4. package/dist/elements/auth-styles.js.map +1 -1
  5. package/dist/elements/minca-auth.d.ts +7 -0
  6. package/dist/elements/minca-auth.d.ts.map +1 -1
  7. package/dist/elements/minca-auth.js +107 -49
  8. package/dist/elements/minca-auth.js.map +1 -1
  9. package/dist/files/file-id.d.ts +51 -0
  10. package/dist/files/file-id.d.ts.map +1 -0
  11. package/dist/files/file-id.js +97 -0
  12. package/dist/files/file-id.js.map +1 -0
  13. package/dist/files/seal.d.ts +20 -0
  14. package/dist/files/seal.d.ts.map +1 -0
  15. package/dist/files/seal.js +77 -0
  16. package/dist/files/seal.js.map +1 -0
  17. package/dist/files/space-files.d.ts +51 -0
  18. package/dist/files/space-files.d.ts.map +1 -0
  19. package/dist/files/space-files.js +476 -0
  20. package/dist/files/space-files.js.map +1 -0
  21. package/dist/index.d.ts +5 -4
  22. package/dist/index.d.ts.map +1 -1
  23. package/dist/index.js +3 -2
  24. package/dist/index.js.map +1 -1
  25. package/dist/node/api.d.ts +6 -1
  26. package/dist/node/api.d.ts.map +1 -1
  27. package/dist/node/api.js +17 -3
  28. package/dist/node/api.js.map +1 -1
  29. package/dist/node/carrier.d.ts +17 -2
  30. package/dist/node/carrier.d.ts.map +1 -1
  31. package/dist/node/carrier.js +26 -4
  32. package/dist/node/carrier.js.map +1 -1
  33. package/dist/node/copy.d.ts +0 -4
  34. package/dist/node/copy.d.ts.map +1 -1
  35. package/dist/node/copy.js +26 -0
  36. package/dist/node/copy.js.map +1 -1
  37. package/dist/node/host.d.ts +66 -1
  38. package/dist/node/host.d.ts.map +1 -1
  39. package/dist/node/host.js +571 -5
  40. package/dist/node/host.js.map +1 -1
  41. package/dist/node/index.d.ts +2 -2
  42. package/dist/node/index.d.ts.map +1 -1
  43. package/dist/node/index.js +1 -1
  44. package/dist/node/index.js.map +1 -1
  45. package/dist/node/node.d.ts.map +1 -1
  46. package/dist/node/node.js +235 -75
  47. package/dist/node/node.js.map +1 -1
  48. package/dist/node/space-runtime.d.ts +9 -0
  49. package/dist/node/space-runtime.d.ts.map +1 -1
  50. package/dist/node/space-runtime.js +125 -16
  51. package/dist/node/space-runtime.js.map +1 -1
  52. package/dist/node/stores.d.ts +44 -8
  53. package/dist/node/stores.d.ts.map +1 -1
  54. package/dist/node/stores.js +44 -7
  55. package/dist/node/stores.js.map +1 -1
  56. package/dist/node/types.d.ts +73 -6
  57. package/dist/node/types.d.ts.map +1 -1
  58. package/dist/node/usage.d.ts +54 -0
  59. package/dist/node/usage.d.ts.map +1 -0
  60. package/dist/node/usage.js +201 -0
  61. package/dist/node/usage.js.map +1 -0
  62. package/dist/node/writer-keys.d.ts +4 -1
  63. package/dist/node/writer-keys.d.ts.map +1 -1
  64. package/dist/node/writer-keys.js +25 -6
  65. package/dist/node/writer-keys.js.map +1 -1
  66. package/dist/node/writers.d.ts.map +1 -1
  67. package/dist/node/writers.js +10 -3
  68. package/dist/node/writers.js.map +1 -1
  69. package/dist/react/minca-auth.d.ts +3 -1
  70. package/dist/react/minca-auth.d.ts.map +1 -1
  71. package/dist/react/minca-auth.js +5 -3
  72. package/dist/react/minca-auth.js.map +1 -1
  73. package/dist/schema/version.d.ts +5 -0
  74. package/dist/schema/version.d.ts.map +1 -1
  75. package/dist/schema/version.js +1 -0
  76. package/dist/schema/version.js.map +1 -1
  77. package/dist/schemas/fragments.d.ts +23 -2
  78. package/dist/schemas/fragments.d.ts.map +1 -1
  79. package/dist/schemas/fragments.js +5 -2
  80. package/dist/schemas/fragments.js.map +1 -1
  81. package/dist/schemas/library/annotations.d.ts +24 -0
  82. package/dist/schemas/library/annotations.d.ts.map +1 -1
  83. package/dist/schemas/library/community.d.ts +24 -0
  84. package/dist/schemas/library/community.d.ts.map +1 -1
  85. package/dist/schemas/library/life.d.ts +72 -0
  86. package/dist/schemas/library/life.d.ts.map +1 -1
  87. package/dist/schemas/library/media.d.ts +144 -0
  88. package/dist/schemas/library/media.d.ts.map +1 -1
  89. package/dist/schemas/library/money.d.ts +24 -0
  90. package/dist/schemas/library/money.d.ts.map +1 -1
  91. package/dist/schemas/library/publishing.d.ts +96 -0
  92. package/dist/schemas/library/publishing.d.ts.map +1 -1
  93. package/dist/schemas/library/social.d.ts +72 -0
  94. package/dist/schemas/library/social.d.ts.map +1 -1
  95. package/dist/session/account-hint.d.ts +13 -0
  96. package/dist/session/account-hint.d.ts.map +1 -0
  97. package/dist/session/account-hint.js +51 -0
  98. package/dist/session/account-hint.js.map +1 -0
  99. package/dist/session/auth.d.ts +51 -3
  100. package/dist/session/auth.d.ts.map +1 -1
  101. package/dist/session/auth.js +262 -36
  102. package/dist/session/auth.js.map +1 -1
  103. package/dist/session/connect.d.ts.map +1 -1
  104. package/dist/session/connect.js +6 -2
  105. package/dist/session/connect.js.map +1 -1
  106. package/dist/session/hosting.d.ts +73 -28
  107. package/dist/session/hosting.d.ts.map +1 -1
  108. package/dist/session/hosting.js +85 -5
  109. package/dist/session/hosting.js.map +1 -1
  110. package/dist/session/index.d.ts +1 -0
  111. package/dist/session/index.d.ts.map +1 -1
  112. package/dist/session/index.js +1 -0
  113. package/dist/session/index.js.map +1 -1
  114. package/dist/session/pairing.d.ts +2 -0
  115. package/dist/session/pairing.d.ts.map +1 -1
  116. package/dist/session/pairing.js +8 -0
  117. package/dist/session/pairing.js.map +1 -1
  118. package/dist/session/places.d.ts +7 -0
  119. package/dist/session/places.d.ts.map +1 -1
  120. package/dist/session/places.js +20 -2
  121. package/dist/session/places.js.map +1 -1
  122. package/dist/space/account-registry.d.ts +13 -0
  123. package/dist/space/account-registry.d.ts.map +1 -1
  124. package/dist/space/account-registry.js +13 -0
  125. package/dist/space/account-registry.js.map +1 -1
  126. package/dist/storage/blob/directory.d.ts +14 -0
  127. package/dist/storage/blob/directory.d.ts.map +1 -0
  128. package/dist/storage/blob/directory.js +80 -0
  129. package/dist/storage/blob/directory.js.map +1 -0
  130. package/dist/storage/folder-adapter.d.ts +22 -0
  131. package/dist/storage/folder-adapter.d.ts.map +1 -1
  132. package/dist/storage/folder-adapter.js +5 -5
  133. package/dist/storage/folder-adapter.js.map +1 -1
  134. package/dist/storage/mirror.d.ts +9 -2
  135. package/dist/storage/mirror.d.ts.map +1 -1
  136. package/dist/storage/mirror.js +60 -35
  137. package/dist/storage/mirror.js.map +1 -1
  138. package/dist/storage/segment.d.ts +7 -2
  139. package/dist/storage/segment.d.ts.map +1 -1
  140. package/dist/storage/segment.js +25 -3
  141. package/dist/storage/segment.js.map +1 -1
  142. package/dist/sync/sync-engine.d.ts +2 -0
  143. package/dist/sync/sync-engine.d.ts.map +1 -1
  144. package/dist/sync/sync-engine.js +5 -0
  145. package/dist/sync/sync-engine.js.map +1 -1
  146. package/dist/sync/sync-messages.d.ts +24 -0
  147. package/dist/sync/sync-messages.d.ts.map +1 -1
  148. package/dist/sync/sync-messages.js.map +1 -1
  149. package/docs/node.md +63 -24
  150. package/docs/sign-in.md +123 -24
  151. package/docs/spaces.md +18 -10
  152. package/docs/standard-library.md +16 -20
  153. package/docs/storage.md +120 -15
  154. package/package.json +1 -1
package/docs/storage.md CHANGED
@@ -1,7 +1,7 @@
1
1
  # Sync and storage in the library
2
2
 
3
3
  How the library paces sync, keeps a store in memory, talks to IndexedDB, a
4
- data folder and a blob store, and mirrors a space into one.
4
+ data folder, a file on disk and a blob store, and mirrors a space into one.
5
5
 
6
6
  > Not protocol. This page describes the reference library, and another
7
7
  > implementation may do it differently and still interoperate. What peers must
@@ -96,7 +96,8 @@ write; without, versions go first, then the entries in one batch, then the
96
96
  deletes, so no entry ever names a version that is not there.
97
97
 
98
98
  Entry keys are strings; values are bytes. The adapters are IndexedDB (below),
99
- the data folder, the sealing wrapper, and an in-memory one for tests.
99
+ a directory (`directoryStores`: the data folder's registry, and a node's
100
+ stores on disk), the sealing wrapper, and an in-memory one for tests.
100
101
 
101
102
  _Source: `packages/core/src/types.ts` (`StorageAdapter`, `BatchOp`). Tests:
102
103
  `packages/core/tests/folder-adapter.test.ts`,
@@ -129,6 +130,10 @@ Each database:
129
130
  range in one `readonly` transaction; `getVersions` is one `readonly`
130
131
  transaction.
131
132
 
133
+ A data folder's own stores ([below](#data-folders)) use the prefix
134
+ `minca-pod:<dataPath with / replaced by :>`: not `minca:`, which is deleted
135
+ when the person forgets an account's browser copy.
136
+
132
137
  Accounts kept in a browser live in database `minca-accounts`, object store
133
138
  `accounts`. A remembered data-folder handle lives in database `minca-folder`,
134
139
  object store `handles`, key `data-folder`.
@@ -146,7 +151,38 @@ adapter needs a browser)._
146
151
 
147
152
  The folder's layout, file names and the way several writers converge are
148
153
  protocol ([spec 05 §14](https://github.com/leifriksheim/minca/blob/main/spec/05-sync-and-storage.md)).
149
- How this library gets and writes one is not:
154
+ How this library keeps and writes one is not.
155
+
156
+ `folderStores(directory, { basePath, localKey, local })` keeps the
157
+ `registry` in the folder, one file per entry, sealed under `localKey`, and
158
+ opens every other path in `local`: a space's entries and bodies, what the
159
+ node keeps beside them, and the `folder/<space id>` store writer state. The
160
+ factory names the folder's segments (`<basePath>/spaces/`, a directory blob
161
+ store) as `folder`, and every space runtime opened from it mirrors to them
162
+ ([mirrors](#mirrors)): it pulls when the space opens and every
163
+ `watchIntervalMs` (default 2 s, `0` to turn it off), and pushes 1 s after a
164
+ change and on close.
165
+
166
+ `local` is this device's:
167
+
168
+ - **In the browser**, IndexedDB under `minca-pod:<dataPath>` (the default).
169
+ - **The CLI** keeps it outside the home, which may be the folder a browser
170
+ uses or one a sync service copies: `$MINCA_LOCAL` or `~/.local/state/minca`,
171
+ then `<first 16 hex of SHA-256 of the home's path>/<account id>/stores`, a
172
+ directory store (`localPath` in `packages/cli/src/home.ts`). A command and
173
+ the daemon share it, and the daemon finds the command's writes as it finds
174
+ any directory store's (`reconcileFolder`). Writer keys are beside it in
175
+ `writers/`; a slot left in the home from before is not read. `minca agent`
176
+ still keeps its key and stores together in `<home>/agent`.
177
+ - **Elsewhere** it must be given: `folderStores` refuses to start without
178
+ one, since a lasting key's places kept in memory would be handed out again
179
+ after a restart.
180
+
181
+ A space kept in an older layout is not read
182
+ ([spec 05 §14.8](https://github.com/leifriksheim/minca/blob/main/spec/05-sync-and-storage.md));
183
+ peers bring it back.
184
+
185
+ Getting the folder:
150
186
 
151
187
  - **In the browser** the folder comes from `showDirectoryPicker` with
152
188
  `id: "minca-pod"`, `mode: "readwrite"`, `startIn: "documents"`, which needs a
@@ -156,18 +192,25 @@ How this library gets and writes one is not:
156
192
  instead. The File System Access API replaces a file atomically on `close()`.
157
193
  - **The CLI** (Node and Bun) writes `<name>.<uuid>.tmp` with mode `0600` and
158
194
  renames it into place. Directories are created with mode `0700`.
159
- - **Re-reading.** While a space is open, the node re-reads its folder every
160
- 2 s (`NodeConfig.watchIntervalMs`, `0` to turn it off) to find what other
161
- writers did.
195
+
196
+ Two folders holding the same account (a copy, a restored backup) share this
197
+ device's local stores, which are named by the account's id: a version known
198
+ to be in one is not pushed to the other. Open one copy at a time, or let the
199
+ other fill in from peers.
162
200
 
163
201
  A folder is the account: copy it to a USB stick and it is all of it, or put
164
202
  it in iCloud, Dropbox or Syncthing and several devices converge with no relay,
165
203
  since the set of versions only grows. Being readable by several origins is the
166
- point, which is why OPFS (origin-private) is not used for it.
204
+ point, which is why OPFS (origin-private) is not used for it. Each device
205
+ adds a folder of segments per space, about one file per 256 KiB of records,
206
+ compacted once it has 32 and twice what the last compaction left.
167
207
 
168
- _Source: `packages/core/src/storage/directory-access.ts`,
169
- `packages/cli/src/fs-directory.ts`, `packages/core/src/node/space-runtime.ts`
170
- (watch loop). Tests: `packages/core/tests/folder-adapter.test.ts`,
208
+ _Source: `packages/core/src/node/stores.ts` (`folderStores`),
209
+ `packages/core/src/storage/blob/directory.ts`,
210
+ `packages/core/src/storage/directory-access.ts`,
211
+ `packages/cli/src/fs-directory.ts`, `packages/cli/src/home.ts` (`unlock`),
212
+ `packages/core/src/node/space-runtime.ts` ("Mirrors", "Writers outside this
213
+ process"). Tests: `packages/core/tests/folder-adapter.test.ts`,
171
214
  `packages/cli/tests/path-safety.test.ts`._
172
215
 
173
216
  ## Blob stores
@@ -186,6 +229,11 @@ and without atomic operations (`BlobStore`):
186
229
  Drivers:
187
230
 
188
231
  - **Memory**, for tests.
232
+ - **Directory**, over a directory handle or a directory on disk
233
+ (`createDirectoryBlobStore`): each `/`-separated part of a key a
234
+ directory, the last a file, every part named as a data folder names its
235
+ files; a directory is removed with the last key in it. What a data folder
236
+ keeps its spaces in.
189
237
  - **S3-compatible** (R2, B2, MinIO, Wasabi, AWS): path-style URLs
190
238
  `<endpoint>/<bucket>/<prefix/><key>`, each `/`-separated part
191
239
  `encodeURIComponent`-ed; SigV4-signed (`aws4fetch`), region default
@@ -196,6 +244,7 @@ Drivers:
196
244
 
197
245
  _Source: `packages/core/src/storage/blob-store.ts`,
198
246
  `packages/core/src/storage/blob/memory.ts`,
247
+ `packages/core/src/storage/blob/directory.ts`,
199
248
  `packages/core/src/storage/blob/s3.ts`. Tests:
200
249
  `packages/core/tests/mirror.test.ts`._
201
250
 
@@ -203,7 +252,7 @@ _Source: `packages/core/src/storage/blob-store.ts`,
203
252
 
204
253
  A mirror's segments, where a store writer may write and how it pulls and compacts
205
254
  are protocol ([spec 05 §16](https://github.com/leifriksheim/minca/blob/main/spec/05-sync-and-storage.md)).
206
- Today carriers and hosts mirror.
255
+ Today data folders, carriers and hosts mirror.
207
256
 
208
257
  A store writer keeps its state in the store `mirrors/<space id>/<n>`, where `n` is
209
258
  the mirror's index:
@@ -218,9 +267,12 @@ the mirror's index:
218
267
  "Known" is everything this store writer uploaded **or read**, so nobody re-uploads
219
268
  everyone else's versions.
220
269
 
221
- A store writer pushes 5 s after the first change and on close, packing what is not
222
- yet known into segments of about 256 KiB (a segment is closed once it reaches
223
- that size). Once it has 32 or more segments of its own, it compacts them.
270
+ A store writer pushes 5 s after the first change (1 s for a data folder) and
271
+ on close, packing what is not yet known into segments of about 256 KiB (a
272
+ segment is closed once it reaches that size). It reads the `mirror:known:`
273
+ entries once, when it starts. Once it has 32 or more segments of its own,
274
+ and twice as many as its last compaction left, it compacts them, keeping
275
+ only what its store holds.
224
276
 
225
277
  _Source: `packages/core/src/storage/mirror.ts`,
226
278
  `packages/core/src/node/space-runtime.ts` ("Mirrors"),
@@ -245,7 +297,6 @@ _Source: `packages/core/src/storage/mirror.ts`,
245
297
  and unicode, a 1 MB blob), with shared backoff on 429 and 5xx that honours
246
298
  `Retry-After`, never retries a delete that returned 404, and never logs a
247
299
  token or signed URL:
248
- - _Directory_: a directory handle or a directory on disk.
249
300
  - _Google Drive_: everything in one app-created folder, `drive.file` scope
250
301
  only (the app sees only its own files). Drive addresses files by id, not
251
302
  name, so the driver keeps a name → file id map from one `files.list`,
@@ -260,3 +311,57 @@ _Source: `packages/core/src/storage/mirror.ts`,
260
311
  - **Budget.** A store like Drive allows about 1,000 requests per 100 seconds.
261
312
  The segment, never the single version, is the unit fetched, and syncing
262
313
  1,000 records must fit that budget.
314
+
315
+ ## Files
316
+
317
+ What peers must agree on (the `files` member, chunks and the id, RFC 8188
318
+ records, the three messages, keeping a file with its body) is in
319
+ [spec 05 §16.6](https://github.com/leifriksheim/minca/blob/main/spec/05-sync-and-storage.md).
320
+ How this library does the rest:
321
+
322
+ - **Adding.** `records.addFile(space, bytes, { mime, name? })` seals the
323
+ bytes under a fresh key and salt, keeps them, and gives back a `FileRef`:
324
+ `{ hash, size, key, salt, mime, name? }`. Put it anywhere in a body;
325
+ writing that body names every reference in it on the version
326
+ (`fileRefsIn`, at most 16), so the envelope and the body always agree.
327
+ - **Reading.** `records.readFile(space, ref)` gives the bytes: held here, or
328
+ fetched from connected peers, one after another, then from mirrors,
329
+ checking every chunk. It asks only for files a version this node keeps
330
+ whole names. For a picture, `URL.createObjectURL(new Blob([bytes], { type:
331
+ ref.mime }))` shows it.
332
+ - **Where chunks live.** Entries in the space's own store: `f/<id>/list`,
333
+ `f/<id>/<n>`, and `f/<id>/own` with the time a file was added and not yet
334
+ named. A data folder keeps these in this device's own stores, not the
335
+ folder.
336
+ - **What a node fetches.** A node with `fetchFiles: 'all'` (carriers and
337
+ hosts) fetches every named file about 2 s after its records change. Any
338
+ other node fetches a file when it is read.
339
+ - **Serving.** A `want-file` is answered with at most 4 chunks; the asker
340
+ asks again for the rest, so a large file never floods a connection.
341
+ - **Keeping.** About 2 s after records change, files no version kept whole
342
+ names are dropped (files of versions that don't stand are not kept),
343
+ except one added here in the last 24 hours and not yet named.
344
+ - **Mirrors.** The same pass uploads each named file this node holds whole,
345
+ with its chunk list, to `files/<space>/<store writer>/`, and deletes this
346
+ writer's copies of files no longer named.
347
+
348
+ _Source: `packages/core/src/files/space-files.ts`, `packages/core/src/files/file-id.ts`,
349
+ `packages/core/src/files/seal.ts`, `packages/core/src/node/space-runtime.ts` ("Files").
350
+ Tests: `packages/core/tests/files.test.ts`._
351
+
352
+ ### Planned: video and pictures
353
+
354
+ > **Planned.** Not normative.
355
+
356
+ - **Video.** `readFile` fetches a whole file. Playing before it is whole
357
+ needs a URL a service worker serves, answering the browser's range
358
+ requests from only the chunks read, opened one at a time (`fileOpener`).
359
+ MP4 with its index first, or WebM; no transcoding; a poster is a second,
360
+ small image file.
361
+ - **Several peers at once.** Chunks asked of several peers in parallel, and
362
+ of a slow one again elsewhere.
363
+ - **Mirrors as sources by range.** `BlobStore.getRange`: a `Range` header on
364
+ S3, `File.slice` in a data folder, so a chunk is read without the whole.
365
+ - **Pictures.** `std.image` gains `width`, `height` and a tiny WebP
366
+ placeholder as a `data:` URL, shown before a chunk arrives; and
367
+ `std.profile`'s `avatar`, resized in the browser before it is added.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@minca/core",
3
- "version": "0.1.7",
3
+ "version": "0.1.9",
4
4
  "type": "module",
5
5
  "description": "Self-sovereign P2P protocol for the browser — identity, schemas, storage, networking, and gossip sync",
6
6
  "repository": {