@wovin/core 0.1.36 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/README.md +0 -12
  2. package/dist/applog/applog-helpers.d.ts +12 -12
  3. package/dist/applog/applog-helpers.d.ts.map +1 -1
  4. package/dist/applog/applog-utils.d.ts +25 -6
  5. package/dist/applog/applog-utils.d.ts.map +1 -1
  6. package/dist/applog/datom-types.d.ts +4 -5
  7. package/dist/applog/datom-types.d.ts.map +1 -1
  8. package/dist/applog.d.ts +3 -3
  9. package/dist/applog.d.ts.map +1 -1
  10. package/dist/{applog.min.js → applog.js} +6 -7
  11. package/dist/blockstore.d.ts +1 -1
  12. package/dist/blockstore.d.ts.map +1 -1
  13. package/dist/{blockstore.min.js → blockstore.js} +1 -3
  14. package/dist/{blockstore.min.js.map → blockstore.js.map} +1 -1
  15. package/dist/{chunk-KXMTKPF4.min.js → chunk-3JZMOEOD.js} +8 -8
  16. package/dist/chunk-3JZMOEOD.js.map +1 -0
  17. package/dist/chunk-3WZVG277.js +434 -0
  18. package/dist/chunk-3WZVG277.js.map +1 -0
  19. package/dist/chunk-7Z5YDQKK.js +1 -0
  20. package/dist/chunk-CPSDKFBG.js +147 -0
  21. package/dist/chunk-CPSDKFBG.js.map +1 -0
  22. package/dist/chunk-E46VTKTZ.js +1 -0
  23. package/dist/{chunk-H3VQJP56.min.js → chunk-J2FDHGOZ.js} +9 -9
  24. package/dist/chunk-J2FDHGOZ.js.map +1 -0
  25. package/dist/chunk-L5EEEGE6.js +1862 -0
  26. package/dist/chunk-L5EEEGE6.js.map +1 -0
  27. package/dist/{chunk-BRC7LSM6.min.js → chunk-PD3C7XUM.js} +5 -5
  28. package/dist/chunk-PD3C7XUM.js.map +1 -0
  29. package/dist/chunk-QZXKQCAY.js +1026 -0
  30. package/dist/chunk-QZXKQCAY.js.map +1 -0
  31. package/dist/{chunk-QPGEBDMJ.min.js → chunk-YDAKBU6Q.js} +1 -1
  32. package/dist/chunk-YDAKBU6Q.js.map +1 -0
  33. package/dist/chunk-ZAADLBSB.js +36 -0
  34. package/dist/chunk-ZAADLBSB.js.map +1 -0
  35. package/dist/index.d.ts +7 -7
  36. package/dist/index.d.ts.map +1 -1
  37. package/dist/{index.min.js → index.js} +73 -46
  38. package/dist/ipfs/car.d.ts +11 -11
  39. package/dist/ipfs/car.d.ts.map +1 -1
  40. package/dist/ipfs/ipfs-utils.d.ts +2 -2
  41. package/dist/ipfs/ipfs-utils.d.ts.map +1 -1
  42. package/dist/ipfs.d.ts +3 -3
  43. package/dist/ipfs.d.ts.map +1 -1
  44. package/dist/{ipfs.min.js → ipfs.js} +7 -10
  45. package/dist/ipns.d.ts +1 -1
  46. package/dist/ipns.d.ts.map +1 -1
  47. package/dist/ipns.js +64 -0
  48. package/dist/ipns.js.map +1 -0
  49. package/dist/pubsub/pub-pull.d.ts +3 -3
  50. package/dist/pubsub/pub-pull.d.ts.map +1 -1
  51. package/dist/pubsub/pubsub-types.d.ts +3 -3
  52. package/dist/pubsub/pubsub-types.d.ts.map +1 -1
  53. package/dist/pubsub/snap-push.d.ts +4 -4
  54. package/dist/pubsub/snap-push.d.ts.map +1 -1
  55. package/dist/pubsub/ucan.d.ts +1 -1
  56. package/dist/pubsub/ucan.d.ts.map +1 -1
  57. package/dist/pubsub.d.ts +4 -4
  58. package/dist/pubsub.d.ts.map +1 -1
  59. package/dist/{pubsub.min.js → pubsub.js} +7 -10
  60. package/dist/query/attr-helpers.d.ts +5 -0
  61. package/dist/query/attr-helpers.d.ts.map +1 -0
  62. package/dist/query/basic.d.ts +85 -21
  63. package/dist/query/basic.d.ts.map +1 -1
  64. package/dist/query/divergences.d.ts +5 -5
  65. package/dist/query/divergences.d.ts.map +1 -1
  66. package/dist/query/entity-collection.d.ts +19 -0
  67. package/dist/query/entity-collection.d.ts.map +1 -0
  68. package/dist/query/matchers.d.ts +1 -1
  69. package/dist/query/matchers.d.ts.map +1 -1
  70. package/dist/query/memoized.d.ts +66 -0
  71. package/dist/query/memoized.d.ts.map +1 -0
  72. package/dist/query/situations.d.ts +2 -1
  73. package/dist/query/situations.d.ts.map +1 -1
  74. package/dist/query/subscribable.d.ts +111 -0
  75. package/dist/query/subscribable.d.ts.map +1 -0
  76. package/dist/query/types.d.ts +54 -14
  77. package/dist/query/types.d.ts.map +1 -1
  78. package/dist/query.d.ts +9 -5
  79. package/dist/query.d.ts.map +1 -1
  80. package/dist/{query.min.js → query.js} +51 -32
  81. package/dist/retrieve/index.d.ts +1 -1
  82. package/dist/retrieve/index.d.ts.map +1 -1
  83. package/dist/retrieve/update-thread.d.ts +3 -3
  84. package/dist/retrieve/update-thread.d.ts.map +1 -1
  85. package/dist/retrieve.d.ts +1 -1
  86. package/dist/retrieve.d.ts.map +1 -1
  87. package/dist/retrieve.js +14 -0
  88. package/dist/thread/basic.d.ts +15 -19
  89. package/dist/thread/basic.d.ts.map +1 -1
  90. package/dist/thread/filters.d.ts +8 -10
  91. package/dist/thread/filters.d.ts.map +1 -1
  92. package/dist/thread/indexes.d.ts +56 -0
  93. package/dist/thread/indexes.d.ts.map +1 -0
  94. package/dist/thread/mapped.d.ts +40 -11
  95. package/dist/thread/mapped.d.ts.map +1 -1
  96. package/dist/thread/utils.d.ts +5 -5
  97. package/dist/thread/utils.d.ts.map +1 -1
  98. package/dist/thread/writeable.d.ts +2 -2
  99. package/dist/thread/writeable.d.ts.map +1 -1
  100. package/dist/thread.d.ts +6 -5
  101. package/dist/thread.d.ts.map +1 -1
  102. package/dist/{thread.min.js → thread.js} +9 -6
  103. package/dist/types/typescript-utils.d.ts +6 -5
  104. package/dist/types/typescript-utils.d.ts.map +1 -1
  105. package/dist/types.d.ts +1 -1
  106. package/dist/types.d.ts.map +1 -1
  107. package/dist/{types.min.js → types.js} +3 -4
  108. package/dist/utils/debug-name.d.ts +13 -0
  109. package/dist/utils/debug-name.d.ts.map +1 -0
  110. package/dist/utils.d.ts +1 -1
  111. package/dist/utils.d.ts.map +1 -1
  112. package/dist/utils.js +9 -0
  113. package/package.json +32 -23
  114. package/src/applog/applog-helpers.ts +155 -0
  115. package/src/applog/applog-utils.test.ts +108 -0
  116. package/src/applog/applog-utils.ts +507 -0
  117. package/src/applog/datom-types.ts +148 -0
  118. package/src/applog.ts +3 -0
  119. package/src/blockstore/index.ts +36 -0
  120. package/src/blockstore.ts +1 -0
  121. package/src/index.ts +8 -0
  122. package/src/ipfs/car.ts +291 -0
  123. package/src/ipfs/fetch-snapshot-chain.ts +135 -0
  124. package/src/ipfs/ipfs-utils.ts +132 -0
  125. package/src/ipfs.ts +3 -0
  126. package/src/ipns/ipns-record.ts +115 -0
  127. package/src/ipns.ts +1 -0
  128. package/src/pubsub/UCAN Specs Overview.md +217 -0
  129. package/src/pubsub/connector.ts +9 -0
  130. package/src/pubsub/pub-pull.ts +31 -0
  131. package/src/pubsub/pubsub-types.ts +90 -0
  132. package/src/pubsub/snap-push.ts +277 -0
  133. package/src/pubsub/ucan-example.ts +61 -0
  134. package/src/pubsub/ucan.ts +56 -0
  135. package/src/pubsub.ts +4 -0
  136. package/src/query/attr-helpers.ts +5 -0
  137. package/src/query/basic.ts +1245 -0
  138. package/src/query/divergences.ts +50 -0
  139. package/src/query/entity-collection.ts +131 -0
  140. package/src/query/liveFilterAndMap.test.ts +102 -0
  141. package/src/query/matchers.ts +8 -0
  142. package/src/query/memoized.test.ts +151 -0
  143. package/src/query/memoized.ts +180 -0
  144. package/src/query/query-steps.ts +4 -0
  145. package/src/query/query.test.ts +538 -0
  146. package/src/query/situations.ts +261 -0
  147. package/src/query/subscribable.test.ts +245 -0
  148. package/src/query/subscribable.ts +234 -0
  149. package/src/query/types.ts +155 -0
  150. package/src/query/withoutDeleted.test.ts +204 -0
  151. package/src/query.ts +9 -0
  152. package/src/retrieve/index.ts +1 -0
  153. package/src/retrieve/update-thread.ts +248 -0
  154. package/src/retrieve.ts +1 -0
  155. package/src/test/perf/query.1m.perf.test.ts +94 -0
  156. package/src/test/perf/query.perf.test.ts +389 -0
  157. package/src/test/perf/query.realdata.perf.test.ts +182 -0
  158. package/src/thread/basic.ts +209 -0
  159. package/src/thread/filters.ts +227 -0
  160. package/src/thread/indexes.ts +250 -0
  161. package/src/thread/joinThreads.test.ts +304 -0
  162. package/src/thread/mapped.ts +226 -0
  163. package/src/thread/utils.ts +144 -0
  164. package/src/thread/writeable.ts +163 -0
  165. package/src/thread.ts +6 -0
  166. package/src/types/typescript-utils.ts +64 -0
  167. package/src/types.ts +1 -0
  168. package/src/utils/debug-name.ts +54 -0
  169. package/src/utils.ts +4 -0
  170. package/dist/chunk-2Y2PYHGR.min.js +0 -65
  171. package/dist/chunk-2Y2PYHGR.min.js.map +0 -1
  172. package/dist/chunk-5MMGBK2U.min.js +0 -1
  173. package/dist/chunk-7IDQIMQO.min.js +0 -1
  174. package/dist/chunk-BRC7LSM6.min.js.map +0 -1
  175. package/dist/chunk-COXXILXC.min.js +0 -512
  176. package/dist/chunk-COXXILXC.min.js.map +0 -1
  177. package/dist/chunk-GDX2OO7L.min.js +0 -9080
  178. package/dist/chunk-GDX2OO7L.min.js.map +0 -1
  179. package/dist/chunk-H3VQJP56.min.js.map +0 -1
  180. package/dist/chunk-HYMC7W6S.min.js +0 -1549
  181. package/dist/chunk-HYMC7W6S.min.js.map +0 -1
  182. package/dist/chunk-KEHU7HGZ.min.js +0 -5216
  183. package/dist/chunk-KEHU7HGZ.min.js.map +0 -1
  184. package/dist/chunk-KXMTKPF4.min.js.map +0 -1
  185. package/dist/chunk-PHITDXZT.min.js +0 -36
  186. package/dist/chunk-QO2KMGDN.min.js +0 -3771
  187. package/dist/chunk-QO2KMGDN.min.js.map +0 -1
  188. package/dist/chunk-QPGEBDMJ.min.js.map +0 -1
  189. package/dist/chunk-WXLCBTHX.min.js +0 -1606
  190. package/dist/chunk-WXLCBTHX.min.js.map +0 -1
  191. package/dist/ipns.min.js +0 -6419
  192. package/dist/ipns.min.js.map +0 -1
  193. package/dist/mobx/mobx-utils.d.ts +0 -82
  194. package/dist/mobx/mobx-utils.d.ts.map +0 -1
  195. package/dist/mobx.d.ts +0 -2
  196. package/dist/mobx.d.ts.map +0 -1
  197. package/dist/mobx.min.js +0 -141
  198. package/dist/retrieve.min.js +0 -17
  199. package/dist/types.min.js.map +0 -1
  200. package/dist/utils.min.js +0 -10
  201. package/dist/utils.min.js.map +0 -1
  202. /package/dist/{applog.min.js.map → applog.js.map} +0 -0
  203. /package/dist/{chunk-5MMGBK2U.min.js.map → chunk-7Z5YDQKK.js.map} +0 -0
  204. /package/dist/{chunk-7IDQIMQO.min.js.map → chunk-E46VTKTZ.js.map} +0 -0
  205. /package/dist/{chunk-PHITDXZT.min.js.map → index.js.map} +0 -0
  206. /package/dist/{index.min.js.map → ipfs.js.map} +0 -0
  207. /package/dist/{ipfs.min.js.map → pubsub.js.map} +0 -0
  208. /package/dist/{mobx.min.js.map → query.js.map} +0 -0
  209. /package/dist/{pubsub.min.js.map → retrieve.js.map} +0 -0
  210. /package/dist/{query.min.js.map → thread.js.map} +0 -0
  211. /package/dist/{retrieve.min.js.map → types.js.map} +0 -0
  212. /package/dist/{thread.min.js.map → utils.js.map} +0 -0
@@ -0,0 +1,148 @@
1
+ // import type { AgentHash } from '../pubsub/pubsub-types.ts'
2
+ // import type { CID } from '@oddjs/odd'
3
+ import { FormatRegistry, Static, TSchema, Type } from '@sinclair/typebox'
4
+ import { TypeCompiler } from '@sinclair/typebox/compiler'
5
+ import { CID } from 'multiformats/cid'
6
+ import type { PartialBy, Tagged } from '../types/typescript-utils.ts'
7
+
8
+ export const Nullable = <T extends TSchema>(schema: T) => Type.Union([schema, Type.Null()])
9
+ export const EntityID_LENGTH = 7
10
+ // const bagu = 'baguqeerav3h4b46j2pyxikqhtm5si5vhzsyrba2duhrtltfutrlmj42anmvq'
11
+ // const k51q = 'k51qzi5uqu5dhe1bxxjxj144bj2a225o1681yobevns26xlxtsfidjgnpwknfd'
12
+ const isCID = /^(k51qz|baguq)[0-9a-z]{56,57}$/ // FIXME: k51 is not really a CID, is it?
13
+ const isShortHash = /^[0-9A-Fa-f]{7,8}$/g // TODO awkward why are some 7 and some 8 long
14
+ // engine level: min 6 (lenient within reason)
15
+ // note3 TBD... either fixed for all entity types VS 7 for pub/sub, 8 for tags, 9 for blocks, 10 for relations etc...
16
+
17
+ FormatRegistry.Set('EntityID', (value) => !!value.match(isShortHash) || !!value.match(isCID))
18
+ export const EntityID = Type.String() // HACK how to configure ID format?
19
+ /*{ format: 'EntityID' }*/
20
+ export type EntityID = Static<typeof EntityID>
21
+
22
+ export type AgentHash = Tagged<string, 'AgentHash'>
23
+ export type DatomPart = string // TODO refactor
24
+ export type CidString = Tagged<string, CID>
25
+ export type IpnsString = Tagged<CidString, 'IPNS'>
26
+ export type AgentID = EntityID
27
+ export type Attribute = string
28
+ export type ApplogValue = string | boolean | number | null // TODO: use Tagged types
29
+ // ? allow objects? or just as serialized strings? Or serialize everything anyways?
30
+
31
+ export interface Atom {
32
+ en: EntityID
33
+ at: Attribute
34
+ vl: ApplogValue
35
+ }
36
+
37
+ export type Timestamp = string
38
+ export interface Applog extends Atom {
39
+ cid: CidString
40
+ pv: CidString | null // ? | CID
41
+ ts: Timestamp
42
+ ag: AgentHash
43
+ }
44
+ export type ApplogNoCid = Omit<Applog, 'cid'>
45
+ export type ApplogOptionalCid = PartialBy<Applog, 'cid'>
46
+ export type ApplogForInsert = PartialBy<ApplogNoCid, 'ts' | 'pv'>
47
+ export type ApplogForInsertOptionalAgent = PartialBy<ApplogForInsert, 'ag'>
48
+
49
+ export interface ApplogEnc {
50
+ cid: CidString
51
+ enc: Uint8Array
52
+ iv?: Uint8Array // iv may be needed if we prefer a different strategy for transmitting iv (odd appends a random iv to the payload via keystoreAES.(en|de)cryptBytes)
53
+ }
54
+ export type ApplogEncNoCid = Omit<ApplogEnc, 'cid'>
55
+
56
+ export type ApplogArrayMaybeEncrypted = (Applog | ApplogEnc)[]
57
+ export type ApplogArrayMaybeEncryptedRO = readonly (Applog | ApplogEnc)[]
58
+ export type ApplogArrayNoCIDMaybeEncryptedRO = readonly (ApplogNoCid | ApplogEncNoCid)[]
59
+
60
+ export type ApplogOfSomeSort = Applog | ApplogEnc | ApplogNoCid | ApplogEncNoCid | ApplogForInsert
61
+
62
+ export const isEncryptedApplog = (l: ApplogOfSomeSort): l is ApplogEnc => (l as ApplogEnc)?.enc instanceof Uint8Array
63
+
64
+ export type AtomPattern = Atom | Applog
65
+
66
+ export interface DatalogStateIdentifier {
67
+ lastTS: Timestamp
68
+ }
69
+
70
+ // New generic type for fields that can be a value, an array of that, or a function
71
+ export type ValueOrMatcher<T> = T | readonly T[] | ReadonlySet<T> | ((value: T) => boolean)
72
+ // Generic type that applies ValueOrMatcher to each field of T
73
+ export type WithMatchers<T extends Record<string, any>> = {
74
+ [K in keyof T & string as `${K}` | `!${K}`]?: ValueOrMatcher<T[K]>
75
+ }
76
+
77
+ export type DatalogQueryPattern = Partial<WithMatchers<Applog>>
78
+ export type DatalogQueryPatternArray = DatalogQueryPattern[]
79
+ export interface DatalogQuery<SELECT extends string> {
80
+ find: readonly SELECT[] // see: https://github.com/microsoft/TypeScript/issues/20965#issuecomment-868981458
81
+ where: DatalogQueryPatternArray
82
+ onlyLatest?: boolean
83
+ }
84
+ export type DatalogQueryResultEntry<SELECT extends string> = Record<
85
+ // SELECT,
86
+ StripPrefix<'?', SELECT>,
87
+ DatomPart
88
+ >
89
+ export type DatalogQueryResultRows<SELECT extends string> = DatalogQueryResultEntry<SELECT>[]
90
+ // export type StripTest = StripPrefix<'?', '?A' | '?B'>
91
+ // export type DatalogQueryResultEntryTEST = DatalogQueryResultEntry<'?A' | '?B'>
92
+ // export type DatalogQueryResultEntryTESTX = MapKeysStripPrefix<'?A' | '?B', '?'>
93
+
94
+ export interface SearchContext {
95
+ [key: string]: ApplogValue
96
+ }
97
+ export interface SearchContextWithLog {
98
+ context: SearchContext
99
+ applog?: Applog
100
+ }
101
+
102
+ export type ResultContext = SearchContext | null
103
+
104
+ /* https://stackoverflow.com/a/72497461 */
105
+ type StripPrefix<
106
+ TPrefix extends string,
107
+ T extends string,
108
+ > = T extends `${TPrefix}${infer R}` ? R : never
109
+
110
+ type MapKeysStripPrefix<SELECT extends string, TPrefix extends string> = {
111
+ [K in SELECT as StripPrefix<TPrefix, K>]: DatomPart
112
+ }
113
+
114
+ FormatRegistry.Set('CID', (value) => !!value.match(isCID))
115
+ export const CIDTB = Type.String({ format: 'CID' })
116
+ export type CIDTB = Static<typeof EntityID>
117
+
118
+ const isURL = /^http([s]?):\/\/.*\..*/
119
+ FormatRegistry.Set('URL', (value) => !!value.match(isURL))
120
+ export const URL = Type.String({ format: 'URL' })
121
+ export type URL = Static<typeof URL>
122
+
123
+ export const AppLogNoCidTB = Type.Object({
124
+ en: EntityID, // EntityID
125
+ at: Type.String(), // Attribute
126
+ vl: Nullable(Type.Union([Type.String(), Type.Boolean(), Type.Number()])), // TODO refactor to semantic typesafe ApplogValue
127
+ ts: Type.String(), // Timestamp
128
+ ag: Type.String(), // AgentHash
129
+ pv: Nullable(CIDTB), // CidString
130
+ })
131
+ export type AppLogNoCidTB = Static<typeof AppLogNoCidTB> // type T = {
132
+
133
+ export const AppLogNoCidTBC = TypeCompiler.Compile(AppLogNoCidTB)
134
+ export const getApplogNoCidTypeErrors = (obj: any) => Array.from(AppLogNoCidTBC.Errors(obj))
135
+ export const isValidApplogNoCid = AppLogNoCidTBC.Check.bind(AppLogNoCidTBC) // ? Include CID
136
+
137
+ export const AppLogTB = Type.Composite([
138
+ Type.Object({
139
+ cid: CIDTB,
140
+ }),
141
+ AppLogNoCidTB,
142
+ ])
143
+ export type AppLogTB = Static<typeof AppLogTB> // type T = {
144
+
145
+ export const AppLogTBC = TypeCompiler.Compile(AppLogTB)
146
+ export const getApplogTypeErrors = (obj: any) => Array.from(AppLogTBC.Errors(obj))
147
+ export const isValidApplog = AppLogTBC.Check.bind(AppLogTBC) // ? Include CID
148
+ // maybe useful for defaulting https://github.com/sinclairzx81/typebox#cast
package/src/applog.ts ADDED
@@ -0,0 +1,3 @@
1
+ export * from './applog/applog-helpers.ts'
2
+ export * from './applog/applog-utils.ts'
3
+ export * from './applog/datom-types.ts'
@@ -0,0 +1,36 @@
1
+ import type { CID } from 'multiformats/cid'
2
+
3
+ /** Minimal async block store interface — get/put/has. */
4
+ export interface BlockStore {
5
+ get(cid: CID): Promise<Uint8Array>
6
+ put(cid: CID, bytes: Uint8Array): Promise<void>
7
+ has(cid: CID): Promise<boolean>
8
+ }
9
+
10
+ /**
11
+ * A block store that reads locally first, with optional remote fallback.
12
+ * On get: local hit → return; local miss + remote → fetch, write-back, return.
13
+ * put/has always operate on local only.
14
+ */
15
+ export class LocalFirstBlockStore implements BlockStore {
16
+ constructor(
17
+ private local: BlockStore,
18
+ private remote?: Pick<BlockStore, 'get'>,
19
+ ) {}
20
+
21
+ async get(cid: CID): Promise<Uint8Array> {
22
+ if (await this.local.has(cid)) return this.local.get(cid)
23
+ if (!this.remote) throw new Error(`Block not found: ${cid}`)
24
+ const bytes = await this.remote.get(cid)
25
+ await this.local.put(cid, bytes)
26
+ return bytes
27
+ }
28
+
29
+ put(cid: CID, bytes: Uint8Array): Promise<void> {
30
+ return this.local.put(cid, bytes)
31
+ }
32
+
33
+ has(cid: CID): Promise<boolean> {
34
+ return this.local.has(cid)
35
+ }
36
+ }
@@ -0,0 +1 @@
1
+ export * from './blockstore/index.ts'
package/src/index.ts ADDED
@@ -0,0 +1,8 @@
1
+ export * from './applog.ts'
2
+ export * from './ipfs.ts'
3
+ export * from './pubsub.ts'
4
+ export * from './query.ts'
5
+ export * from './retrieve.ts'
6
+ export * from './thread.ts'
7
+ export * from './types.ts'
8
+ // export * from './utils.ts'
@@ -0,0 +1,291 @@
1
+ import { CarReader, CarWriter } from '@ipld/car'
2
+ import * as dagJson from '@ipld/dag-json'
3
+ import { Logger } from 'besonders-logger'
4
+ import { BlockView, CID } from 'multiformats'
5
+ import { sortApplogsByTs } from '../applog/applog-utils.ts'
6
+ import { Applog, ApplogArrayMaybeEncrypted, CidString } from '../applog/datom-types.ts'
7
+ import { unchunkApplogsBlock } from '../pubsub/snap-push.ts'
8
+ import { SnapBlockLogs, SnapBlockLogsOrChunks, SnapRootBlock } from '../pubsub/pubsub-types.ts'
9
+ import { areCidsEqual, containsCid } from './ipfs-utils.ts'
10
+
11
+ const { WARN, LOG, DEBUG, VERBOSE, ERROR } = Logger.setup(Logger.INFO) // eslint-disable-line no-unused-vars
12
+
13
+ export type CIDForCar = CID // Exclude<Parameters<(typeof CarWriter)['create']>[0], void>
14
+ export type BlockForCar = Parameters<CarWriter['put']>[0]
15
+
16
+ export interface BlockStoreish {
17
+ get(cid: CID): PromiseLike<Uint8Array> // (i) not using decoded version to be similar to blockstore-idb
18
+ }
19
+
20
+ export interface DecodedCar {
21
+ rootCID: CID
22
+ // blocks: Map<CidString, any>
23
+ blockStore: BlockStoreish
24
+ }
25
+
26
+ /** Warning: unsorted & maybe encrypted */
27
+ export async function decodePubFromCar(car: CarReader) {
28
+ const decoded = await getBlocksOfCar(car)
29
+ return await decodePubFromBlocks(decoded)
30
+ }
31
+
32
+ export async function decodePubFromBlocks(
33
+ { rootCID, blockStore }: DecodedCar,
34
+ _recursionTrace: CID[] = [], // DEPRECATED: kept for API compat, unused in iterative version
35
+ stopAtCID?: CID // NEW: stop iteration when we hit this CID
36
+ ) {
37
+ if (!rootCID || !blockStore) {
38
+ throw ERROR('Empty roots/blocks', { rootCID, blockStore })
39
+ }
40
+
41
+ let allApplogs: ApplogArrayMaybeEncrypted = []
42
+ let firstInfo: { logs: CID[] } | null = null
43
+ let currentCID: CID | undefined = rootCID
44
+ const visited = new Set<string>() // Loop detection (replaces recursionTrace)
45
+ let applogsCID: CID | null = null // From first snapshot only
46
+
47
+ while (currentCID) {
48
+ const cidStr = currentCID.toString()
49
+
50
+ // Loop detection
51
+ if (visited.has(cidStr)) {
52
+ throw ERROR('[decodePubFromBlocks] pub chain has a loop', {
53
+ currentCID: cidStr,
54
+ visited: [...visited]
55
+ })
56
+ }
57
+ visited.add(cidStr)
58
+
59
+ // Decode current snapshot
60
+ const root = (await getDecodedBlock(blockStore, currentCID)) as SnapRootBlock
61
+ VERBOSE(`[decodePubFromBlocks] root:`, cidStr, root, { blockStore })
62
+ if (!root) {
63
+ throw ERROR('[decodePubFromBlocks] root not found in blockStore', { blockStore, currentCID: cidStr })
64
+ }
65
+
66
+ // Decode applogs for this snapshot
67
+ let pubLogsArray: CID[]
68
+ if (root?.info) {
69
+ // New(er) format
70
+ if (!applogsCID) applogsCID = root.applogs // Save from first snapshot
71
+ const applogsBlock = (await getDecodedBlock(blockStore, root.applogs)) as SnapBlockLogsOrChunks
72
+ pubLogsArray = await unchunkApplogsBlock(applogsBlock, blockStore)
73
+ // Info only from first (most recent) snapshot
74
+ if (!firstInfo) {
75
+ firstInfo = (await getDecodedBlock(blockStore, root.info)) as SnapBlockLogs
76
+ DEBUG(`new format - infoLogs`, firstInfo.logs.map(l => ({ [l.toString()]: l })))
77
+ }
78
+ // TODO: verify signatures
79
+ } else {
80
+ // Old format
81
+ pubLogsArray = root.applogs as any as CID[]
82
+ }
83
+
84
+ const resolveLogFromCidLink = async (cidOrLink: CID) => {
85
+ const cid = cidOrLink
86
+ const applog = (await getDecodedBlock(blockStore, cid)) as Applog
87
+ if (!applog) {
88
+ ERROR(`Could not find applog CID in pub blocks:`, cid.toString(), { cid, root, blockStore })
89
+ throw new Error(`Could not find applog CID in pub blocks: ${cid.toString()}`)
90
+ }
91
+ if ((applog.pv as any) instanceof CID) applog.pv = (applog.pv as any as CID).toV1().toString()
92
+ return {
93
+ ...applog,
94
+ cid: cid.toV1().toString(),
95
+ }
96
+ }
97
+
98
+ const snapshotApplogs = await Promise.all(pubLogsArray.map(resolveLogFromCidLink))
99
+ allApplogs = allApplogs.concat(snapshotApplogs)
100
+
101
+ // Check if we should stop
102
+ if (!root.prev) {
103
+ break // End of chain
104
+ }
105
+ if (stopAtCID && areCidsEqual(root.prev, stopAtCID)) {
106
+ DEBUG('[decodePubFromBlocks] stopping at stopAtCID:', stopAtCID.toString())
107
+ break // Reached already-pulled snapshot
108
+ }
109
+
110
+ // Verify prev exists before continuing
111
+ const prevBytes = await blockStore.get(root.prev)
112
+ if (!prevBytes) {
113
+ throw ERROR('[decodePubFromBlocks] prev snapshot missing from blockStore', {
114
+ currentCID: cidStr,
115
+ prev: root.prev.toString(),
116
+ stopAtCID: stopAtCID?.toString(),
117
+ visited: [...visited]
118
+ })
119
+ }
120
+
121
+ currentCID = root.prev // Move to previous snapshot
122
+ }
123
+
124
+ const result = {
125
+ cid: rootCID,
126
+ info: firstInfo ? {
127
+ ...firstInfo,
128
+ logs: await Promise.all(firstInfo.logs.map(async (cidOrLink: CID) => {
129
+ const cid = cidOrLink
130
+ const applog = (await getDecodedBlock(blockStore, cid)) as Applog
131
+ if (!applog) {
132
+ ERROR(`Could not find info log CID in pub blocks:`, cid.toString(), { cid, blockStore })
133
+ throw new Error(`Could not find info log CID in pub blocks: ${cid.toString()}`)
134
+ }
135
+ if ((applog.pv as any) instanceof CID) applog.pv = (applog.pv as any as CID).toV1().toString()
136
+ return {
137
+ ...applog,
138
+ cid: cid.toV1().toString(),
139
+ }
140
+ })),
141
+ } : null,
142
+ applogsCID,
143
+ applogs: allApplogs,
144
+ }
145
+ DEBUG('[decodePubFromBlocks] result:', result, { rootCID: rootCID.toString(), blockStore, applogs: allApplogs })
146
+ return result
147
+ }
148
+
149
+ export async function getBlocksOfCar(car: CarReader) {
150
+ const rootsFromCar = await car.getRoots()
151
+ const roots = rootsFromCar.map(c => ((typeof c.toV1 === 'function') ? c : CID.decode(c.bytes)).toV1().toString() as CidString) // HACK
152
+ const blocks = new Map<CidString, any>()
153
+ for await (const { cid: cidFromCarblocks, bytes } of car.blocks()) {
154
+ const cid = (typeof cidFromCarblocks.toV1 === 'function') ? cidFromCarblocks : CID.decode(cidFromCarblocks.bytes)
155
+ VERBOSE({ cidFromCarblocks, cid })
156
+ // blocks.set(cid.toV1().toString(), dagJson.decode(bytes)) // HACK: tried using CID as map key, but because it's based on referential equality it's not working
157
+ blocks.set(cid.toV1().toString(), bytes) // HACK: tried using CID as map key, but because it's based on referential equality it's not working
158
+ }
159
+ if (roots.length !== 1) {
160
+ WARN('Unexpected roots count:', roots)
161
+ }
162
+ return {
163
+ rootCID: CID.parse(roots[0]),
164
+ blockStore: {
165
+ get: (cid) => blocks.get(cid.toV1().toString()),
166
+ },
167
+ } satisfies DecodedCar
168
+ }
169
+ export async function getDecodedBlock(blockStore: BlockStoreish, cid: CID) {
170
+ try {
171
+ var blob = await blockStore.get(cid)
172
+ if (!blob) {
173
+ WARN('returning null')
174
+ return null // I don't think this ever happens actually
175
+ }
176
+ } catch (err) {
177
+ if ((err as any).message === 'Not Found') return null
178
+ throw err
179
+ }
180
+ return dagJson.decode(blob)
181
+ }
182
+
183
+ // make out in the car... been a while but also sounds nice
184
+ export async function makeCarOut(roots: CIDForCar, blocks: BlockForCar[]) {
185
+ const { writer, out } = CarWriter.create(Array.isArray(roots) ? roots : [roots])
186
+
187
+ // add the blocks to the CAR and close it
188
+ VERBOSE(`Writing ${blocks.length} blocks to CAR`, { roots, blocks })
189
+ blocks.forEach(b => writer.put(b))
190
+ writer.close()
191
+ // VERBOSE(`Wrote ${blocks.length} blocks to CAR`, writer)
192
+ return out
193
+ } /** create a new CarWriter, with the encoded block as the root */
194
+
195
+ // export async function makeCarReader(roots: CIDForCar, blocks: BlockForCar[]) {
196
+ // const out = await makeCarOut(roots, blocks)
197
+
198
+ // // create a new CarReader we can hand to web3.storage.putCar
199
+ // const reader = await CarReader.fromIterable(out)
200
+ // VERBOSE(`CAR reader`, reader)
201
+ // return reader
202
+ // } /** create a new CarWriter, with the encoded block as the root */
203
+
204
+ export async function makeCarBlob(roots: CIDForCar, blocks: BlockForCar[]) {
205
+ const carOut = await makeCarOut(roots, blocks)
206
+ const chunks = []
207
+ for await (const chunk of carOut) {
208
+ chunks.push(chunk)
209
+ }
210
+ const blob = new Blob(chunks)
211
+ return blob
212
+ }
213
+ export async function carFromBlob(blob: Blob | File): Promise<CarReader> {
214
+ return CarReader.fromBytes(new Uint8Array(await blob.arrayBuffer()))
215
+ }
216
+
217
+ function extractCids(value: unknown): CID[] {
218
+ if (value instanceof CID) return [value]
219
+ if (Array.isArray(value)) return value.flatMap(extractCids)
220
+ if (value && typeof value === 'object') return Object.values(value).flatMap(extractCids)
221
+ return []
222
+ }
223
+
224
+ const MAX_COLLECT_BLOCKS = 1_000_000
225
+
226
+ export async function collectDagBlocks(
227
+ startCID: CID,
228
+ blockStore: BlockStoreish,
229
+ ): Promise<BlockForCar[]> {
230
+ const visited = new Set<string>()
231
+ const blocks: BlockForCar[] = []
232
+ const queue: CID[] = [startCID]
233
+
234
+ while (queue.length > 0) {
235
+ if (blocks.length >= MAX_COLLECT_BLOCKS) {
236
+ WARN(`[collectDagBlocks] hit ${MAX_COLLECT_BLOCKS} block limit, returning partial result`)
237
+ break
238
+ }
239
+
240
+ const cid = queue.shift()!
241
+ const cidStr = cid.toString()
242
+ if (visited.has(cidStr)) continue
243
+ visited.add(cidStr)
244
+
245
+ let bytes: Uint8Array
246
+ try {
247
+ bytes = await blockStore.get(cid)
248
+ } catch {
249
+ WARN(`[collectDagBlocks] block not found: ${cidStr}, stopping this branch`)
250
+ continue
251
+ }
252
+ if (!bytes) {
253
+ WARN(`[collectDagBlocks] block not found: ${cidStr}, stopping this branch`)
254
+ continue
255
+ }
256
+
257
+ blocks.push({ cid, bytes })
258
+
259
+ if (blocks.length % 1000 === 0) {
260
+ LOG(`[collectDagBlocks] collected ${blocks.length} blocks...`)
261
+ }
262
+
263
+ try {
264
+ const decoded = dagJson.decode(bytes)
265
+ const childCids = extractCids(decoded)
266
+ for (const child of childCids) {
267
+ if (!visited.has(child.toString())) {
268
+ queue.push(child)
269
+ }
270
+ }
271
+ } catch {
272
+ // Not dag-json — leaf block, no children to walk
273
+ }
274
+ }
275
+
276
+ DEBUG(`[collectDagBlocks] collected ${blocks.length} blocks from ${startCID.toString()}`)
277
+ return blocks
278
+ }
279
+
280
+ export function streamReaderToIterable(bodyReader: ReadableStreamDefaultReader<Uint8Array>): AsyncIterable<Uint8Array> {
281
+ return (async function*() {
282
+ while (true) {
283
+ const { done, value } = await bodyReader.read()
284
+ VERBOSE(`[car] chunk`, { done, value })
285
+ if (done) {
286
+ break
287
+ }
288
+ yield value
289
+ }
290
+ })()
291
+ }
@@ -0,0 +1,135 @@
1
+ import { CarReader } from '@ipld/car'
2
+ import * as dagJson from '@ipld/dag-json'
3
+ import { Logger } from 'besonders-logger'
4
+ import { CID } from 'multiformats/cid'
5
+ import type { SnapRootBlock } from '../pubsub/pubsub-types.ts'
6
+ import { areCidsEqual } from './ipfs-utils.ts'
7
+
8
+ const { WARN, LOG, DEBUG, VERBOSE, ERROR } = Logger.setup(Logger.INFO) // eslint-disable-line no-unused-vars
9
+
10
+ export interface BlockStoreForFetch {
11
+ get(cid: CID): Promise<Uint8Array | undefined>
12
+ put(cid: CID, bytes: Uint8Array): Promise<void>
13
+ }
14
+
15
+ export interface FetchChainOptions {
16
+ rootCID: CID
17
+ stopAtCID?: CID // Stop when we hit this CID (lastCID from subscription)
18
+ stopAtCounter?: number // Stop when we reach this counter (walking backwards)
19
+ fetchBlock: (cid: CID) => Promise<CarReader> // dag-scope=block
20
+ fetchAll: (cid: CID) => Promise<CarReader> // dag-scope=all
21
+ maxDepth?: number
22
+ }
23
+
24
+ export interface FetchChainResult {
25
+ rootCID: CID
26
+ blockStore: BlockStoreForFetch
27
+ /** Serializable blocks array for worker boundary crossing */
28
+ blocks: [string, Uint8Array][]
29
+ snapshotCount: number
30
+ counterRange?: { minCounter: number; maxCounter: number }
31
+ }
32
+
33
+ /**
34
+ * Fetches a snapshot chain iteratively, stopping at stopAtCID.
35
+ * Uses 3 requests per snapshot: root(block), applogs(all), info(all).
36
+ * This avoids the gateway's dag-scope=all following prev links recursively.
37
+ */
38
+ export async function fetchSnapshotChainUntil(options: FetchChainOptions): Promise<FetchChainResult> {
39
+ const { rootCID, stopAtCID, stopAtCounter, fetchBlock, fetchAll, maxDepth = 100 } = options
40
+ const blockStore = createMemoryBlockStore()
41
+ const visited = new Set<string>() // Loop detection for fetch
42
+ let currentCID: CID | undefined = rootCID
43
+ let snapshotCount = 0
44
+ let minCounter = Infinity
45
+ let maxCounter = -Infinity
46
+
47
+ while (currentCID && snapshotCount < maxDepth) {
48
+ const cidStr = currentCID.toString()
49
+
50
+ // Loop detection
51
+ if (visited.has(cidStr)) {
52
+ throw ERROR('[fetchSnapshotChain] snapshot chain has a loop', { currentCID: cidStr, visited: [...visited] })
53
+ }
54
+ visited.add(cidStr)
55
+
56
+ // Check stop condition BEFORE fetching content
57
+ if (stopAtCID && areCidsEqual(currentCID, stopAtCID)) {
58
+ DEBUG('[fetchSnapshotChain] reached stopAtCID, stopping', stopAtCID.toString())
59
+ break // We've reached the last pulled snapshot - don't fetch it again
60
+ }
61
+
62
+ // 1. Fetch root block only (dag-scope=block)
63
+ DEBUG('[fetchSnapshotChain] fetching root block', cidStr)
64
+ const rootCar = await fetchBlock(currentCID)
65
+ await addCarBlocksToStore(rootCar, blockStore)
66
+
67
+ // Parse root to get applogs, info, prev CIDs
68
+ const rootBytes = await blockStore.get(currentCID)
69
+ if (!rootBytes) {
70
+ throw ERROR('[fetchSnapshotChain] root block not in store after fetch', { currentCID: cidStr })
71
+ }
72
+ const root = dagJson.decode(rootBytes) as SnapRootBlock
73
+
74
+ // Track counter range
75
+ if (typeof root.prevCounter === 'number') {
76
+ minCounter = Math.min(minCounter, root.prevCounter)
77
+ maxCounter = Math.max(maxCounter, root.prevCounter)
78
+ }
79
+
80
+ // Stop condition based on counter
81
+ if (stopAtCounter !== undefined && typeof root.prevCounter === 'number' && root.prevCounter <= stopAtCounter) {
82
+ DEBUG('[fetchSnapshotChain] reached stopAtCounter', { stopAtCounter, prevCounter: root.prevCounter })
83
+ break
84
+ }
85
+
86
+ // 2. Fetch applogs with dag-scope=all (gets applogs block + all linked applog blocks)
87
+ DEBUG('[fetchSnapshotChain] fetching applogs', root.applogs.toString())
88
+ const applogsCar = await fetchAll(root.applogs)
89
+ await addCarBlocksToStore(applogsCar, blockStore)
90
+
91
+ // 3. Fetch info with dag-scope=all (gets info block + all linked info log blocks)
92
+ DEBUG('[fetchSnapshotChain] fetching info', root.info.toString())
93
+ const infoCar = await fetchAll(root.info)
94
+ await addCarBlocksToStore(infoCar, blockStore)
95
+
96
+ snapshotCount++
97
+ currentCID = root.prev // Move to previous snapshot
98
+ }
99
+
100
+ DEBUG('[fetchSnapshotChain] done', { snapshotCount, rootCID: rootCID.toString() })
101
+ return {
102
+ rootCID,
103
+ blockStore,
104
+ blocks: blockStore.getBlocksArray(),
105
+ snapshotCount,
106
+ counterRange: minCounter !== Infinity ? { minCounter, maxCounter } : undefined,
107
+ }
108
+ }
109
+
110
+ async function addCarBlocksToStore(car: CarReader, store: BlockStoreForFetch) {
111
+ for await (const { cid, bytes } of car.blocks()) {
112
+ const validCid = typeof cid.toV1 === 'function' ? cid : CID.decode(cid.bytes)
113
+ await store.put(validCid, bytes)
114
+ }
115
+ }
116
+
117
+ interface MemoryBlockStoreWithBlocks extends BlockStoreForFetch {
118
+ /** Get all blocks as serializable array */
119
+ getBlocksArray(): [string, Uint8Array][]
120
+ }
121
+
122
+ function createMemoryBlockStore(): MemoryBlockStoreWithBlocks {
123
+ const blocks = new Map<string, Uint8Array>()
124
+ return {
125
+ async get(cid: CID) {
126
+ return blocks.get(cid.toV1().toString())
127
+ },
128
+ async put(cid: CID, bytes: Uint8Array) {
129
+ blocks.set(cid.toV1().toString(), bytes)
130
+ },
131
+ getBlocksArray() {
132
+ return Array.from(blocks.entries())
133
+ },
134
+ }
135
+ }