@wovin/core 0.0.0-ciao-mobx-955482e8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/LICENSE +661 -0
  2. package/README.md +3 -0
  3. package/dist/applog/applog-helpers.d.ts +47 -0
  4. package/dist/applog/applog-helpers.d.ts.map +1 -0
  5. package/dist/applog/applog-utils.d.ts +57 -0
  6. package/dist/applog/applog-utils.d.ts.map +1 -0
  7. package/dist/applog/datom-types.d.ts +128 -0
  8. package/dist/applog/datom-types.d.ts.map +1 -0
  9. package/dist/applog.d.ts +4 -0
  10. package/dist/applog.d.ts.map +1 -0
  11. package/dist/applog.js +101 -0
  12. package/dist/applog.js.map +1 -0
  13. package/dist/blockstore/index.d.ts +21 -0
  14. package/dist/blockstore/index.d.ts.map +1 -0
  15. package/dist/blockstore.d.ts +2 -0
  16. package/dist/blockstore.d.ts.map +1 -0
  17. package/dist/blockstore.js +24 -0
  18. package/dist/blockstore.js.map +1 -0
  19. package/dist/chunk-6MQKRL6W.js +86 -0
  20. package/dist/chunk-6MQKRL6W.js.map +1 -0
  21. package/dist/chunk-7MW34UEO.js +40 -0
  22. package/dist/chunk-7MW34UEO.js.map +1 -0
  23. package/dist/chunk-7Z5YDQKK.js +1 -0
  24. package/dist/chunk-7Z5YDQKK.js.map +1 -0
  25. package/dist/chunk-CY4NLISM.js +144 -0
  26. package/dist/chunk-CY4NLISM.js.map +1 -0
  27. package/dist/chunk-E46VTKTZ.js +1 -0
  28. package/dist/chunk-E46VTKTZ.js.map +1 -0
  29. package/dist/chunk-O43W7UW6.js +434 -0
  30. package/dist/chunk-O43W7UW6.js.map +1 -0
  31. package/dist/chunk-XIQSYEV3.js +1604 -0
  32. package/dist/chunk-XIQSYEV3.js.map +1 -0
  33. package/dist/chunk-XVGW4QC3.js +55 -0
  34. package/dist/chunk-XVGW4QC3.js.map +1 -0
  35. package/dist/chunk-YDAKBU6Q.js +9 -0
  36. package/dist/chunk-YDAKBU6Q.js.map +1 -0
  37. package/dist/chunk-ZAADLBSB.js +36 -0
  38. package/dist/chunk-ZAADLBSB.js.map +1 -0
  39. package/dist/chunk-ZXCJRYD7.js +883 -0
  40. package/dist/chunk-ZXCJRYD7.js.map +1 -0
  41. package/dist/index.d.ts +8 -0
  42. package/dist/index.d.ts.map +1 -0
  43. package/dist/index.js +354 -0
  44. package/dist/index.js.map +1 -0
  45. package/dist/ipfs/car.d.ts +59 -0
  46. package/dist/ipfs/car.d.ts.map +1 -0
  47. package/dist/ipfs/fetch-snapshot-chain.d.ts +32 -0
  48. package/dist/ipfs/fetch-snapshot-chain.d.ts.map +1 -0
  49. package/dist/ipfs/ipfs-utils.d.ts +35 -0
  50. package/dist/ipfs/ipfs-utils.d.ts.map +1 -0
  51. package/dist/ipfs.d.ts +4 -0
  52. package/dist/ipfs.d.ts.map +1 -0
  53. package/dist/ipfs.js +60 -0
  54. package/dist/ipfs.js.map +1 -0
  55. package/dist/ipns/ipns-record.d.ts +34 -0
  56. package/dist/ipns/ipns-record.d.ts.map +1 -0
  57. package/dist/ipns.d.ts +2 -0
  58. package/dist/ipns.d.ts.map +1 -0
  59. package/dist/ipns.js +64 -0
  60. package/dist/ipns.js.map +1 -0
  61. package/dist/pubsub/connector.d.ts +9 -0
  62. package/dist/pubsub/connector.d.ts.map +1 -0
  63. package/dist/pubsub/pub-pull.d.ts +14 -0
  64. package/dist/pubsub/pub-pull.d.ts.map +1 -0
  65. package/dist/pubsub/pubsub-types.d.ts +72 -0
  66. package/dist/pubsub/pubsub-types.d.ts.map +1 -0
  67. package/dist/pubsub/snap-push.d.ts +41 -0
  68. package/dist/pubsub/snap-push.d.ts.map +1 -0
  69. package/dist/pubsub/ucan-example.d.ts +3 -0
  70. package/dist/pubsub/ucan-example.d.ts.map +1 -0
  71. package/dist/pubsub/ucan.d.ts +16 -0
  72. package/dist/pubsub/ucan.d.ts.map +1 -0
  73. package/dist/pubsub.d.ts +5 -0
  74. package/dist/pubsub.d.ts.map +1 -0
  75. package/dist/pubsub.js +31 -0
  76. package/dist/pubsub.js.map +1 -0
  77. package/dist/query/basic.d.ts +105 -0
  78. package/dist/query/basic.d.ts.map +1 -0
  79. package/dist/query/divergences.d.ts +12 -0
  80. package/dist/query/divergences.d.ts.map +1 -0
  81. package/dist/query/matchers.d.ts +4 -0
  82. package/dist/query/matchers.d.ts.map +1 -0
  83. package/dist/query/memoized.d.ts +66 -0
  84. package/dist/query/memoized.d.ts.map +1 -0
  85. package/dist/query/query-steps.d.ts +4 -0
  86. package/dist/query/query-steps.d.ts.map +1 -0
  87. package/dist/query/situations.d.ts +80 -0
  88. package/dist/query/situations.d.ts.map +1 -0
  89. package/dist/query/subscribable.d.ts +102 -0
  90. package/dist/query/subscribable.d.ts.map +1 -0
  91. package/dist/query/types.d.ts +70 -0
  92. package/dist/query/types.d.ts.map +1 -0
  93. package/dist/query.d.ts +8 -0
  94. package/dist/query.d.ts.map +1 -0
  95. package/dist/query.js +108 -0
  96. package/dist/query.js.map +1 -0
  97. package/dist/retrieve/index.d.ts +2 -0
  98. package/dist/retrieve/index.d.ts.map +1 -0
  99. package/dist/retrieve/update-thread.d.ts +64 -0
  100. package/dist/retrieve/update-thread.d.ts.map +1 -0
  101. package/dist/retrieve.d.ts +2 -0
  102. package/dist/retrieve.d.ts.map +1 -0
  103. package/dist/retrieve.js +14 -0
  104. package/dist/retrieve.js.map +1 -0
  105. package/dist/thread/basic.d.ts +60 -0
  106. package/dist/thread/basic.d.ts.map +1 -0
  107. package/dist/thread/filters.d.ts +47 -0
  108. package/dist/thread/filters.d.ts.map +1 -0
  109. package/dist/thread/mapped.d.ts +31 -0
  110. package/dist/thread/mapped.d.ts.map +1 -0
  111. package/dist/thread/utils.d.ts +23 -0
  112. package/dist/thread/utils.d.ts.map +1 -0
  113. package/dist/thread/writeable.d.ts +41 -0
  114. package/dist/thread/writeable.d.ts.map +1 -0
  115. package/dist/thread.d.ts +6 -0
  116. package/dist/thread.d.ts.map +1 -0
  117. package/dist/thread.js +54 -0
  118. package/dist/thread.js.map +1 -0
  119. package/dist/types/typescript-utils.d.ts +34 -0
  120. package/dist/types/typescript-utils.d.ts.map +1 -0
  121. package/dist/types.d.ts +2 -0
  122. package/dist/types.d.ts.map +1 -0
  123. package/dist/types.js +26 -0
  124. package/dist/types.js.map +1 -0
  125. package/dist/utils/debug-name.d.ts +13 -0
  126. package/dist/utils/debug-name.d.ts.map +1 -0
  127. package/dist/utils.d.ts +4 -0
  128. package/dist/utils.d.ts.map +1 -0
  129. package/dist/utils.js +9 -0
  130. package/dist/utils.js.map +1 -0
  131. package/package.json +110 -0
  132. package/src/applog/applog-helpers.ts +150 -0
  133. package/src/applog/applog-utils.ts +398 -0
  134. package/src/applog/datom-types.ts +148 -0
  135. package/src/applog.ts +3 -0
  136. package/src/blockstore/index.ts +36 -0
  137. package/src/blockstore.ts +1 -0
  138. package/src/index.ts +8 -0
  139. package/src/ipfs/car.ts +291 -0
  140. package/src/ipfs/fetch-snapshot-chain.ts +135 -0
  141. package/src/ipfs/ipfs-utils.ts +132 -0
  142. package/src/ipfs.ts +3 -0
  143. package/src/ipns/ipns-record.ts +115 -0
  144. package/src/ipns.ts +1 -0
  145. package/src/pubsub/UCAN Specs Overview.md +217 -0
  146. package/src/pubsub/connector.ts +9 -0
  147. package/src/pubsub/pub-pull.ts +31 -0
  148. package/src/pubsub/pubsub-types.ts +90 -0
  149. package/src/pubsub/snap-push.ts +277 -0
  150. package/src/pubsub/ucan-example.ts +61 -0
  151. package/src/pubsub/ucan.ts +56 -0
  152. package/src/pubsub.ts +4 -0
  153. package/src/query/basic.ts +1061 -0
  154. package/src/query/divergences.ts +50 -0
  155. package/src/query/matchers.ts +8 -0
  156. package/src/query/memoized.test.ts +151 -0
  157. package/src/query/memoized.ts +180 -0
  158. package/src/query/query-steps.ts +4 -0
  159. package/src/query/query.test.ts +536 -0
  160. package/src/query/situations.ts +261 -0
  161. package/src/query/subscribable.test.ts +245 -0
  162. package/src/query/subscribable.ts +225 -0
  163. package/src/query/types.ts +155 -0
  164. package/src/query.ts +7 -0
  165. package/src/retrieve/index.ts +1 -0
  166. package/src/retrieve/update-thread.ts +248 -0
  167. package/src/retrieve.ts +1 -0
  168. package/src/test/perf/query.1m.perf.test.ts +94 -0
  169. package/src/test/perf/query.perf.test.ts +389 -0
  170. package/src/test/perf/query.realdata.perf.test.ts +175 -0
  171. package/src/thread/basic.ts +209 -0
  172. package/src/thread/filters.ts +234 -0
  173. package/src/thread/mapped.ts +166 -0
  174. package/src/thread/utils.ts +146 -0
  175. package/src/thread/writeable.ts +163 -0
  176. package/src/thread.ts +5 -0
  177. package/src/types/typescript-utils.ts +64 -0
  178. package/src/types.ts +1 -0
  179. package/src/utils/debug-name.ts +54 -0
  180. package/src/utils.ts +4 -0
@@ -0,0 +1,291 @@
1
+ import { CarReader, CarWriter } from '@ipld/car'
2
+ import * as dagJson from '@ipld/dag-json'
3
+ import { Logger } from 'besonders-logger'
4
+ import { BlockView, CID } from 'multiformats'
5
+ import { sortApplogsByTs } from '../applog/applog-utils.ts'
6
+ import { Applog, ApplogArrayMaybeEncrypted, CidString } from '../applog/datom-types.ts'
7
+ import { unchunkApplogsBlock } from '../pubsub/snap-push.ts'
8
+ import { SnapBlockLogs, SnapBlockLogsOrChunks, SnapRootBlock } from '../pubsub/pubsub-types.ts'
9
+ import { areCidsEqual, containsCid } from './ipfs-utils.ts'
10
+
11
+ const { WARN, LOG, DEBUG, VERBOSE, ERROR } = Logger.setup(Logger.INFO) // eslint-disable-line no-unused-vars
12
+
13
+ export type CIDForCar = CID // Exclude<Parameters<(typeof CarWriter)['create']>[0], void>
14
+ export type BlockForCar = Parameters<CarWriter['put']>[0]
15
+
16
+ export interface BlockStoreish {
17
+ get(cid: CID): PromiseLike<Uint8Array> // (i) not using decoded version to be similar to blockstore-idb
18
+ }
19
+
20
+ export interface DecodedCar {
21
+ rootCID: CID
22
+ // blocks: Map<CidString, any>
23
+ blockStore: BlockStoreish
24
+ }
25
+
26
+ /** Warning: unsorted & maybe encrypted */
27
+ export async function decodePubFromCar(car: CarReader) {
28
+ const decoded = await getBlocksOfCar(car)
29
+ return await decodePubFromBlocks(decoded)
30
+ }
31
+
32
+ export async function decodePubFromBlocks(
33
+ { rootCID, blockStore }: DecodedCar,
34
+ _recursionTrace: CID[] = [], // DEPRECATED: kept for API compat, unused in iterative version
35
+ stopAtCID?: CID // NEW: stop iteration when we hit this CID
36
+ ) {
37
+ if (!rootCID || !blockStore) {
38
+ throw ERROR('Empty roots/blocks', { rootCID, blockStore })
39
+ }
40
+
41
+ let allApplogs: ApplogArrayMaybeEncrypted = []
42
+ let firstInfo: { logs: CID[] } | null = null
43
+ let currentCID: CID | undefined = rootCID
44
+ const visited = new Set<string>() // Loop detection (replaces recursionTrace)
45
+ let applogsCID: CID | null = null // From first snapshot only
46
+
47
+ while (currentCID) {
48
+ const cidStr = currentCID.toString()
49
+
50
+ // Loop detection
51
+ if (visited.has(cidStr)) {
52
+ throw ERROR('[decodePubFromBlocks] pub chain has a loop', {
53
+ currentCID: cidStr,
54
+ visited: [...visited]
55
+ })
56
+ }
57
+ visited.add(cidStr)
58
+
59
+ // Decode current snapshot
60
+ const root = (await getDecodedBlock(blockStore, currentCID)) as SnapRootBlock
61
+ VERBOSE(`[decodePubFromBlocks] root:`, cidStr, root, { blockStore })
62
+ if (!root) {
63
+ throw ERROR('[decodePubFromBlocks] root not found in blockStore', { blockStore, currentCID: cidStr })
64
+ }
65
+
66
+ // Decode applogs for this snapshot
67
+ let pubLogsArray: CID[]
68
+ if (root?.info) {
69
+ // New(er) format
70
+ if (!applogsCID) applogsCID = root.applogs // Save from first snapshot
71
+ const applogsBlock = (await getDecodedBlock(blockStore, root.applogs)) as SnapBlockLogsOrChunks
72
+ pubLogsArray = await unchunkApplogsBlock(applogsBlock, blockStore)
73
+ // Info only from first (most recent) snapshot
74
+ if (!firstInfo) {
75
+ firstInfo = (await getDecodedBlock(blockStore, root.info)) as SnapBlockLogs
76
+ DEBUG(`new format - infoLogs`, firstInfo.logs.map(l => ({ [l.toString()]: l })))
77
+ }
78
+ // TODO: verify signatures
79
+ } else {
80
+ // Old format
81
+ pubLogsArray = root.applogs as any as CID[]
82
+ }
83
+
84
+ const resolveLogFromCidLink = async (cidOrLink: CID) => {
85
+ const cid = cidOrLink
86
+ const applog = (await getDecodedBlock(blockStore, cid)) as Applog
87
+ if (!applog) {
88
+ ERROR(`Could not find applog CID in pub blocks:`, cid.toString(), { cid, root, blockStore })
89
+ throw new Error(`Could not find applog CID in pub blocks: ${cid.toString()}`)
90
+ }
91
+ if ((applog.pv as any) instanceof CID) applog.pv = (applog.pv as any as CID).toV1().toString()
92
+ return {
93
+ ...applog,
94
+ cid: cid.toV1().toString(),
95
+ }
96
+ }
97
+
98
+ const snapshotApplogs = await Promise.all(pubLogsArray.map(resolveLogFromCidLink))
99
+ allApplogs = allApplogs.concat(snapshotApplogs)
100
+
101
+ // Check if we should stop
102
+ if (!root.prev) {
103
+ break // End of chain
104
+ }
105
+ if (stopAtCID && areCidsEqual(root.prev, stopAtCID)) {
106
+ DEBUG('[decodePubFromBlocks] stopping at stopAtCID:', stopAtCID.toString())
107
+ break // Reached already-pulled snapshot
108
+ }
109
+
110
+ // Verify prev exists before continuing
111
+ const prevBytes = await blockStore.get(root.prev)
112
+ if (!prevBytes) {
113
+ throw ERROR('[decodePubFromBlocks] prev snapshot missing from blockStore', {
114
+ currentCID: cidStr,
115
+ prev: root.prev.toString(),
116
+ stopAtCID: stopAtCID?.toString(),
117
+ visited: [...visited]
118
+ })
119
+ }
120
+
121
+ currentCID = root.prev // Move to previous snapshot
122
+ }
123
+
124
+ const result = {
125
+ cid: rootCID,
126
+ info: firstInfo ? {
127
+ ...firstInfo,
128
+ logs: await Promise.all(firstInfo.logs.map(async (cidOrLink: CID) => {
129
+ const cid = cidOrLink
130
+ const applog = (await getDecodedBlock(blockStore, cid)) as Applog
131
+ if (!applog) {
132
+ ERROR(`Could not find info log CID in pub blocks:`, cid.toString(), { cid, blockStore })
133
+ throw new Error(`Could not find info log CID in pub blocks: ${cid.toString()}`)
134
+ }
135
+ if ((applog.pv as any) instanceof CID) applog.pv = (applog.pv as any as CID).toV1().toString()
136
+ return {
137
+ ...applog,
138
+ cid: cid.toV1().toString(),
139
+ }
140
+ })),
141
+ } : null,
142
+ applogsCID,
143
+ applogs: allApplogs,
144
+ }
145
+ DEBUG('[decodePubFromBlocks] result:', result, { rootCID: rootCID.toString(), blockStore, applogs: allApplogs })
146
+ return result
147
+ }
148
+
149
+ export async function getBlocksOfCar(car: CarReader) {
150
+ const rootsFromCar = await car.getRoots()
151
+ const roots = rootsFromCar.map(c => ((typeof c.toV1 === 'function') ? c : CID.decode(c.bytes)).toV1().toString() as CidString) // HACK
152
+ const blocks = new Map<CidString, any>()
153
+ for await (const { cid: cidFromCarblocks, bytes } of car.blocks()) {
154
+ const cid = (typeof cidFromCarblocks.toV1 === 'function') ? cidFromCarblocks : CID.decode(cidFromCarblocks.bytes)
155
+ VERBOSE({ cidFromCarblocks, cid })
156
+ // blocks.set(cid.toV1().toString(), dagJson.decode(bytes)) // HACK: tried using CID as map key, but because it's based on referential equality it's not working
157
+ blocks.set(cid.toV1().toString(), bytes) // HACK: tried using CID as map key, but because it's based on referential equality it's not working
158
+ }
159
+ if (roots.length !== 1) {
160
+ WARN('Unexpected roots count:', roots)
161
+ }
162
+ return {
163
+ rootCID: CID.parse(roots[0]),
164
+ blockStore: {
165
+ get: (cid) => blocks.get(cid.toV1().toString()),
166
+ },
167
+ } satisfies DecodedCar
168
+ }
169
+ export async function getDecodedBlock(blockStore: BlockStoreish, cid: CID) {
170
+ try {
171
+ var blob = await blockStore.get(cid)
172
+ if (!blob) {
173
+ WARN('returning null')
174
+ return null // I don't think this ever happens actually
175
+ }
176
+ } catch (err) {
177
+ if ((err as any).message === 'Not Found') return null
178
+ throw err
179
+ }
180
+ return dagJson.decode(blob)
181
+ }
182
+
183
+ // make out in the car... been a while but also sounds nice
184
+ export async function makeCarOut(roots: CIDForCar, blocks: BlockForCar[]) {
185
+ const { writer, out } = CarWriter.create(Array.isArray(roots) ? roots : [roots])
186
+
187
+ // add the blocks to the CAR and close it
188
+ VERBOSE(`Writing ${blocks.length} blocks to CAR`, { roots, blocks })
189
+ blocks.forEach(b => writer.put(b))
190
+ writer.close()
191
+ // VERBOSE(`Wrote ${blocks.length} blocks to CAR`, writer)
192
+ return out
193
+ } /** create a new CarWriter, with the encoded block as the root */
194
+
195
+ // export async function makeCarReader(roots: CIDForCar, blocks: BlockForCar[]) {
196
+ // const out = await makeCarOut(roots, blocks)
197
+
198
+ // // create a new CarReader we can hand to web3.storage.putCar
199
+ // const reader = await CarReader.fromIterable(out)
200
+ // VERBOSE(`CAR reader`, reader)
201
+ // return reader
202
+ // } /** create a new CarWriter, with the encoded block as the root */
203
+
204
+ export async function makeCarBlob(roots: CIDForCar, blocks: BlockForCar[]) {
205
+ const carOut = await makeCarOut(roots, blocks)
206
+ const chunks = []
207
+ for await (const chunk of carOut) {
208
+ chunks.push(chunk)
209
+ }
210
+ const blob = new Blob(chunks)
211
+ return blob
212
+ }
213
+ export async function carFromBlob(blob: Blob | File): Promise<CarReader> {
214
+ return CarReader.fromBytes(new Uint8Array(await blob.arrayBuffer()))
215
+ }
216
+
217
+ function extractCids(value: unknown): CID[] {
218
+ if (value instanceof CID) return [value]
219
+ if (Array.isArray(value)) return value.flatMap(extractCids)
220
+ if (value && typeof value === 'object') return Object.values(value).flatMap(extractCids)
221
+ return []
222
+ }
223
+
224
+ const MAX_COLLECT_BLOCKS = 1_000_000
225
+
226
+ export async function collectDagBlocks(
227
+ startCID: CID,
228
+ blockStore: BlockStoreish,
229
+ ): Promise<BlockForCar[]> {
230
+ const visited = new Set<string>()
231
+ const blocks: BlockForCar[] = []
232
+ const queue: CID[] = [startCID]
233
+
234
+ while (queue.length > 0) {
235
+ if (blocks.length >= MAX_COLLECT_BLOCKS) {
236
+ WARN(`[collectDagBlocks] hit ${MAX_COLLECT_BLOCKS} block limit, returning partial result`)
237
+ break
238
+ }
239
+
240
+ const cid = queue.shift()!
241
+ const cidStr = cid.toString()
242
+ if (visited.has(cidStr)) continue
243
+ visited.add(cidStr)
244
+
245
+ let bytes: Uint8Array
246
+ try {
247
+ bytes = await blockStore.get(cid)
248
+ } catch {
249
+ WARN(`[collectDagBlocks] block not found: ${cidStr}, stopping this branch`)
250
+ continue
251
+ }
252
+ if (!bytes) {
253
+ WARN(`[collectDagBlocks] block not found: ${cidStr}, stopping this branch`)
254
+ continue
255
+ }
256
+
257
+ blocks.push({ cid, bytes })
258
+
259
+ if (blocks.length % 1000 === 0) {
260
+ LOG(`[collectDagBlocks] collected ${blocks.length} blocks...`)
261
+ }
262
+
263
+ try {
264
+ const decoded = dagJson.decode(bytes)
265
+ const childCids = extractCids(decoded)
266
+ for (const child of childCids) {
267
+ if (!visited.has(child.toString())) {
268
+ queue.push(child)
269
+ }
270
+ }
271
+ } catch {
272
+ // Not dag-json — leaf block, no children to walk
273
+ }
274
+ }
275
+
276
+ DEBUG(`[collectDagBlocks] collected ${blocks.length} blocks from ${startCID.toString()}`)
277
+ return blocks
278
+ }
279
+
280
+ export function streamReaderToIterable(bodyReader: ReadableStreamDefaultReader<Uint8Array>): AsyncIterable<Uint8Array> {
281
+ return (async function*() {
282
+ while (true) {
283
+ const { done, value } = await bodyReader.read()
284
+ VERBOSE(`[car] chunk`, { done, value })
285
+ if (done) {
286
+ break
287
+ }
288
+ yield value
289
+ }
290
+ })()
291
+ }
@@ -0,0 +1,135 @@
1
+ import { CarReader } from '@ipld/car'
2
+ import * as dagJson from '@ipld/dag-json'
3
+ import { Logger } from 'besonders-logger'
4
+ import { CID } from 'multiformats/cid'
5
+ import type { SnapRootBlock } from '../pubsub/pubsub-types.ts'
6
+ import { areCidsEqual } from './ipfs-utils.ts'
7
+
8
+ const { WARN, LOG, DEBUG, VERBOSE, ERROR } = Logger.setup(Logger.INFO) // eslint-disable-line no-unused-vars
9
+
10
+ export interface BlockStoreForFetch {
11
+ get(cid: CID): Promise<Uint8Array | undefined>
12
+ put(cid: CID, bytes: Uint8Array): Promise<void>
13
+ }
14
+
15
+ export interface FetchChainOptions {
16
+ rootCID: CID
17
+ stopAtCID?: CID // Stop when we hit this CID (lastCID from subscription)
18
+ stopAtCounter?: number // Stop when we reach this counter (walking backwards)
19
+ fetchBlock: (cid: CID) => Promise<CarReader> // dag-scope=block
20
+ fetchAll: (cid: CID) => Promise<CarReader> // dag-scope=all
21
+ maxDepth?: number
22
+ }
23
+
24
+ export interface FetchChainResult {
25
+ rootCID: CID
26
+ blockStore: BlockStoreForFetch
27
+ /** Serializable blocks array for worker boundary crossing */
28
+ blocks: [string, Uint8Array][]
29
+ snapshotCount: number
30
+ counterRange?: { minCounter: number; maxCounter: number }
31
+ }
32
+
33
+ /**
34
+ * Fetches a snapshot chain iteratively, stopping at stopAtCID.
35
+ * Uses 3 requests per snapshot: root(block), applogs(all), info(all).
36
+ * This avoids the gateway's dag-scope=all following prev links recursively.
37
+ */
38
+ export async function fetchSnapshotChainUntil(options: FetchChainOptions): Promise<FetchChainResult> {
39
+ const { rootCID, stopAtCID, stopAtCounter, fetchBlock, fetchAll, maxDepth = 100 } = options
40
+ const blockStore = createMemoryBlockStore()
41
+ const visited = new Set<string>() // Loop detection for fetch
42
+ let currentCID: CID | undefined = rootCID
43
+ let snapshotCount = 0
44
+ let minCounter = Infinity
45
+ let maxCounter = -Infinity
46
+
47
+ while (currentCID && snapshotCount < maxDepth) {
48
+ const cidStr = currentCID.toString()
49
+
50
+ // Loop detection
51
+ if (visited.has(cidStr)) {
52
+ throw ERROR('[fetchSnapshotChain] snapshot chain has a loop', { currentCID: cidStr, visited: [...visited] })
53
+ }
54
+ visited.add(cidStr)
55
+
56
+ // Check stop condition BEFORE fetching content
57
+ if (stopAtCID && areCidsEqual(currentCID, stopAtCID)) {
58
+ DEBUG('[fetchSnapshotChain] reached stopAtCID, stopping', stopAtCID.toString())
59
+ break // We've reached the last pulled snapshot - don't fetch it again
60
+ }
61
+
62
+ // 1. Fetch root block only (dag-scope=block)
63
+ DEBUG('[fetchSnapshotChain] fetching root block', cidStr)
64
+ const rootCar = await fetchBlock(currentCID)
65
+ await addCarBlocksToStore(rootCar, blockStore)
66
+
67
+ // Parse root to get applogs, info, prev CIDs
68
+ const rootBytes = await blockStore.get(currentCID)
69
+ if (!rootBytes) {
70
+ throw ERROR('[fetchSnapshotChain] root block not in store after fetch', { currentCID: cidStr })
71
+ }
72
+ const root = dagJson.decode(rootBytes) as SnapRootBlock
73
+
74
+ // Track counter range
75
+ if (typeof root.prevCounter === 'number') {
76
+ minCounter = Math.min(minCounter, root.prevCounter)
77
+ maxCounter = Math.max(maxCounter, root.prevCounter)
78
+ }
79
+
80
+ // Stop condition based on counter
81
+ if (stopAtCounter !== undefined && typeof root.prevCounter === 'number' && root.prevCounter <= stopAtCounter) {
82
+ DEBUG('[fetchSnapshotChain] reached stopAtCounter', { stopAtCounter, prevCounter: root.prevCounter })
83
+ break
84
+ }
85
+
86
+ // 2. Fetch applogs with dag-scope=all (gets applogs block + all linked applog blocks)
87
+ DEBUG('[fetchSnapshotChain] fetching applogs', root.applogs.toString())
88
+ const applogsCar = await fetchAll(root.applogs)
89
+ await addCarBlocksToStore(applogsCar, blockStore)
90
+
91
+ // 3. Fetch info with dag-scope=all (gets info block + all linked info log blocks)
92
+ DEBUG('[fetchSnapshotChain] fetching info', root.info.toString())
93
+ const infoCar = await fetchAll(root.info)
94
+ await addCarBlocksToStore(infoCar, blockStore)
95
+
96
+ snapshotCount++
97
+ currentCID = root.prev // Move to previous snapshot
98
+ }
99
+
100
+ DEBUG('[fetchSnapshotChain] done', { snapshotCount, rootCID: rootCID.toString() })
101
+ return {
102
+ rootCID,
103
+ blockStore,
104
+ blocks: blockStore.getBlocksArray(),
105
+ snapshotCount,
106
+ counterRange: minCounter !== Infinity ? { minCounter, maxCounter } : undefined,
107
+ }
108
+ }
109
+
110
+ async function addCarBlocksToStore(car: CarReader, store: BlockStoreForFetch) {
111
+ for await (const { cid, bytes } of car.blocks()) {
112
+ const validCid = typeof cid.toV1 === 'function' ? cid : CID.decode(cid.bytes)
113
+ await store.put(validCid, bytes)
114
+ }
115
+ }
116
+
117
+ interface MemoryBlockStoreWithBlocks extends BlockStoreForFetch {
118
+ /** Get all blocks as serializable array */
119
+ getBlocksArray(): [string, Uint8Array][]
120
+ }
121
+
122
+ function createMemoryBlockStore(): MemoryBlockStoreWithBlocks {
123
+ const blocks = new Map<string, Uint8Array>()
124
+ return {
125
+ async get(cid: CID) {
126
+ return blocks.get(cid.toV1().toString())
127
+ },
128
+ async put(cid: CID, bytes: Uint8Array) {
129
+ blocks.set(cid.toV1().toString(), bytes)
130
+ },
131
+ getBlocksArray() {
132
+ return Array.from(blocks.entries())
133
+ },
134
+ }
135
+ }
@@ -0,0 +1,132 @@
1
+ import * as dagJson from '@ipld/dag-json'
2
+ import { sha256 } from '@noble/hashes/sha2.js'
3
+ import { Logger } from 'besonders-logger'
4
+ import { CID, digest as Digest } from 'multiformats'
5
+ import { encode as multiformatsEncode } from 'multiformats/block'
6
+ // import { encode } from 'multiformats/block';
7
+ import { Applog, ApplogEncNoCid, ApplogNoCid, ApplogOfSomeSort, CidString, IpnsString, isEncryptedApplog } from '../applog/datom-types.ts'
8
+
9
+ import { base36 } from 'multiformats/bases/base36'
10
+ import { sha256 as sha265Hasher } from 'multiformats/hashes/sha2'
11
+
12
+ /* THIS FILE SHOULD NOT DEPEND ON UI STUFF, SO THAT TESTS CAN RUN WITH MINIMAL DEPENDENCIES */
13
+
14
+ const { WARN, LOG, DEBUG, VERBOSE, ERROR } = Logger.setup(Logger.INFO) // eslint-disable-line no-unused-vars
15
+
16
+ export const MULTICODEC_IPNS_KEY = 0x72
17
+
18
+ export function prepareForPub(log: ApplogOfSomeSort, without: string[] = ['cid']) {
19
+ if (!log) throw ERROR('falsy log', log)
20
+ let cid = (log as Applog).cid
21
+ if (isEncryptedApplog(log)) {
22
+ if (!cid) cid = getCidSync(encodeBlock(log as ApplogEncNoCid).bytes).toString()
23
+ WARN('preparing an encrypted applog - really?')
24
+ return { log, cid }
25
+ }
26
+ const logWithout = {}
27
+ for (let [key, val] of Object.entries(log)) {
28
+ if (val === undefined) {
29
+ WARN(`log.${key} is undefined, which is not allowed - encoding as null`, log)
30
+ val = null
31
+ }
32
+ if (!without.includes(key)) {
33
+ logWithout[key] = val // && key === 'pv' ? CID.parse(val) : val //HACK: disabled until clarified: https://discuss.ipfs.tech/t/pin-dag-with-open-ends/17612
34
+ } else {
35
+ VERBOSE('excluding app log', { key, val })
36
+ }
37
+ }
38
+ return { log: logWithout as Applog, cid }
39
+ }
40
+
41
+ export function encodeApplogAndGetCid(log: ApplogNoCid) {
42
+ return getCidSync(encodeApplog(log).bytes)
43
+ }
44
+ export function encodeApplog(log: ApplogNoCid | ApplogEncNoCid): { bytes: dagJson.ByteView<any>; cid: CID } {
45
+ return encodeBlock(prepareForPub(log)?.log)
46
+ }
47
+
48
+ export function getCidSync(bytes: dagJson.ByteView<any>) {
49
+ // Hacky way to use a sync sha265 lib to create a CID - code inspired by https://github.com/multiformats/js-multiformats#multihash-hashers
50
+ const hash = sha256(bytes)
51
+ const digest = Digest.create(sha265Hasher.code, hash)
52
+ const cid = CID.create(1, dagJson.code, digest)
53
+ VERBOSE(`[getCidSync]`, { bytes, hash, digest, cid })
54
+ return cid
55
+ }
56
+ /** encode the json object into an IPLD block */
57
+ export function encodeBlock(jsonObject: any): { bytes: dagJson.ByteView<any>; cid: CID } {
58
+ DEBUG('[encodeBlock]', jsonObject)
59
+ try {
60
+ const byteView = dagJson.encode(jsonObject)
61
+ return { bytes: byteView, cid: getCidSync(byteView) }
62
+ } catch (err) {
63
+ throw ERROR('[encodeBlock] failed to encode:', jsonObject, err)
64
+ }
65
+ }
66
+
67
+ export async function encodeBlockOriginal(jsonObject: any) {
68
+ // HACK re-added this to verify the sync variant is sane
69
+ const encoded = await multiformatsEncode({ value: jsonObject, codec: dagJson, hasher: sha265Hasher })
70
+ const syncVariant = encodeBlock(jsonObject)
71
+ if (syncVariant.cid.toString() !== encoded.cid.toString()) {
72
+ ERROR(`[encodeBlockOriginal] sync cid mismatch`, { jsonObject, encoded, syncVariant })
73
+ }
74
+ return encoded
75
+ }
76
+
77
+ export function tryParseCID(cidString: CidString) {
78
+ let cid: CID | null = null
79
+ let errors = []
80
+ try {
81
+ cid = CID.parse(cidString)
82
+ } catch (err) {
83
+ VERBOSE(`[retrieveThread] couldn't parse pubID with default base`)
84
+ errors.push(err)
85
+ }
86
+ if (!cid) {
87
+ try {
88
+ cid = CID.parse(cidString, base36) // e.g. for IPNS
89
+ } catch (err) {
90
+ VERBOSE(`[retrieveThread] couldn't parse pubID with base36`)
91
+ errors.push(err)
92
+ }
93
+ }
94
+ return {
95
+ cid,
96
+ errors: cid ? null : errors, // we only care about errors if we failed to parse
97
+ isIpns: cid && isIpnsKeyCid(cid),
98
+ }
99
+ }
100
+ export function isIpnsKeyCid(cid: CID) {
101
+ return cid.code === MULTICODEC_IPNS_KEY
102
+ }
103
+
104
+ export function cidToString(cid: CID) {
105
+ if (cid.code == MULTICODEC_IPNS_KEY) {
106
+ return toIpnsString(cid)
107
+ } else {
108
+ return cid.toString()
109
+ }
110
+ }
111
+ export function toIpnsString(cid: CID) {
112
+ if (cid.code !== MULTICODEC_IPNS_KEY) throw ERROR(`Not an IPNS cid (${cid.code}):`, cid.toString())
113
+ return cid.toString(base36) as IpnsString
114
+ }
115
+ export function ensureValidCIDinstance(cidOrStringA: CID | CidString) {
116
+ return typeof cidOrStringA === 'string'
117
+ ? CID.parse(cidOrStringA)
118
+ : typeof cidOrStringA.toV1 != 'function'
119
+ ? CID.decode(cidOrStringA.bytes)
120
+ : cidOrStringA
121
+ }
122
+ export function areCidsEqual(cidOrStringA: CID | CidString, cidOrStringB: CID | CidString) {
123
+ if (!cidOrStringA || !cidOrStringB) throw new Error(`[areCidsEqual] invalid params: ${cidOrStringA}, ${cidOrStringB}`)
124
+ if (cidOrStringA === cidOrStringB) return true // shortcut if both are strings
125
+ const cidA = ensureValidCIDinstance(cidOrStringA)
126
+ const cidB = ensureValidCIDinstance(cidOrStringB)
127
+ return cidA.toV1().toString() === cidB.toV1().toString()
128
+ }
129
+ export function containsCid(list: (CID | CidString)[] | Set<CidString>, needle: CID | CidString) {
130
+ if (list instanceof Set) return list.has(typeof needle === 'string' ? needle : needle.toV1().toString()) // ? what if the CidString is a different form? (parse and format would cost performance)
131
+ return list.some(cidOrString => areCidsEqual(cidOrString, needle))
132
+ }
package/src/ipfs.ts ADDED
@@ -0,0 +1,3 @@
1
+ export * from './ipfs/car.ts'
2
+ export * from './ipfs/ipfs-utils.ts'
3
+ export * from './ipfs/fetch-snapshot-chain.ts'
@@ -0,0 +1,115 @@
1
+ import { createIPNSRecord, marshalIPNSRecord, unmarshalIPNSRecord } from 'ipns'
2
+ import { privateKeyFromRaw } from '@libp2p/crypto/keys'
3
+ import { base36 } from 'multiformats/bases/base36'
4
+ import { base64pad } from 'multiformats/bases/base64'
5
+ import type { CID } from 'multiformats/cid'
6
+
7
+ export interface SignedIPNSRecord {
8
+ recordBytes: Uint8Array // marshalled protobuf, signed — the wire format
9
+ ipnsName: string // k51... string
10
+ value: string // /ipfs/<cid>
11
+ sequence: bigint
12
+ }
13
+
14
+ /** Derive IPNS name string (k51...) from Ed25519 private key bytes */
15
+ export function ipnsNameFromPrivateKey(privKeyBytes: Uint8Array): string {
16
+ const privKey = privateKeyFromRaw(privKeyBytes)
17
+ return privKey.publicKey.toCID().toString(base36)
18
+ }
19
+
20
+ /**
21
+ * Create a signed IPNS record (protobuf wire format).
22
+ * Same bytes can be published to any naming service.
23
+ */
24
+ export async function createSignedIPNSRecord(
25
+ privateKey: Uint8Array,
26
+ cid: CID,
27
+ sequence: bigint,
28
+ lifetimeMs = 365 * 24 * 60 * 60 * 1000, // 1 year
29
+ ): Promise<SignedIPNSRecord> {
30
+ const privKey = privateKeyFromRaw(privateKey)
31
+ const value = `/ipfs/${cid.toV1()}`
32
+ const record = await createIPNSRecord(privKey, value, sequence, lifetimeMs)
33
+ const recordBytes = marshalIPNSRecord(record)
34
+ const ipnsName = privKey.publicKey.toCID().toString(base36)
35
+ return { recordBytes, ipnsName, value, sequence }
36
+ }
37
+
38
+ export { unmarshalIPNSRecord }
39
+
40
+ /** A target that can receive a signed IPNS record */
41
+ export interface IPNSPublishTarget {
42
+ name: string
43
+ publish(ipnsName: string, recordBytes: Uint8Array): Promise<void>
44
+ }
45
+
46
+ /**
47
+ * Resolve current IPNS sequence number from a naming service.
48
+ * Returns null if the name was never published (404).
49
+ * Throws on network/server errors.
50
+ */
51
+ export async function resolveIPNSSequence(
52
+ nameServiceUrl: string,
53
+ ipnsName: string,
54
+ ): Promise<bigint | null> {
55
+ const url = `${nameServiceUrl}/name/${ipnsName}`
56
+ let response: Response
57
+ try {
58
+ response = await fetch(url)
59
+ } catch (err) {
60
+ throw new Error(`Network error resolving IPNS ${ipnsName}: ${err}`)
61
+ }
62
+
63
+ if (response.status === 404) return null
64
+ if (!response.ok) {
65
+ throw new Error(`HTTP ${response.status} resolving IPNS ${ipnsName}: ${response.statusText}`)
66
+ }
67
+
68
+ const { record, value } = await response.json()
69
+ if (!record && !value) return null // never published
70
+
71
+ // If raw record is available, unmarshal to get sequence
72
+ if (record) {
73
+ const bytes = base64pad.baseDecode(record)
74
+ const entry = unmarshalIPNSRecord(bytes)
75
+ return entry.sequence
76
+ }
77
+
78
+ // Some servers return value but not raw record — can't get sequence
79
+ return 0n
80
+ }
81
+
82
+ /**
83
+ * Create a signed IPNS record and publish to all configured targets.
84
+ * Resolves sequence from sequenceServiceUrl, creates record once, fans out.
85
+ * Throws if ALL targets fail; warns on partial failure.
86
+ */
87
+ export async function publishIPNSRecord(
88
+ privateKey: Uint8Array,
89
+ cid: CID,
90
+ targets: IPNSPublishTarget[],
91
+ sequenceServiceUrl = 'https://name.web3.storage',
92
+ ): Promise<SignedIPNSRecord> {
93
+ const ipnsName = ipnsNameFromPrivateKey(privateKey)
94
+ const currentSeq = await resolveIPNSSequence(sequenceServiceUrl, ipnsName)
95
+ const sequence = currentSeq != null ? currentSeq + 1n : 0n
96
+ const signed = await createSignedIPNSRecord(privateKey, cid, sequence)
97
+
98
+ const results = await Promise.allSettled(
99
+ targets.map(t => t.publish(ipnsName, signed.recordBytes))
100
+ )
101
+ const failures = results
102
+ .map((r, i) => ({ r, name: targets[i].name }))
103
+ .filter(({ r }) => r.status === 'rejected') as { r: PromiseRejectedResult; name: string }[]
104
+
105
+ if (failures.length > 0 && failures.length < targets.length) {
106
+ // Partial failure — log but don't throw
107
+ for (const { r, name } of failures) {
108
+ console.warn(`[publishIPNSRecord] target '${name}' failed:`, r.reason)
109
+ }
110
+ } else if (failures.length === targets.length) {
111
+ throw new Error(`All IPNS publish targets failed: ${failures.map(({ r, name }) => `${name}: ${r.reason}`).join('; ')}`)
112
+ }
113
+
114
+ return signed
115
+ }
package/src/ipns.ts ADDED
@@ -0,0 +1 @@
1
+ export * from './ipns/ipns-record.ts'