@wovin/core 0.1.36 → 0.2.2

package/src/pubsub/pubsub-types.ts

@@ -0,0 +1,90 @@
+import { CID } from 'multiformats/cid'
+import { cyrb53hash } from './../applog/applog-utils.ts'
+import { AgentHash, AgentID, CidString } from '../applog/datom-types.ts'
+import { Tagged } from '../types.ts'
+import { UCANCapMap } from './ucan.ts'
+type AgentString = Tagged<string, 'AgentString'>
+type DIDString = Tagged<string, 'DID'>
+export type { AgentHash, AgentString, DIDString }
+
+
+export interface AppAgent {
+	ag: AgentHash
+	agentString: AgentString
+	did: DIDString
+	sign? (data: Uint8Array): Promise<Uint8Array>
+	ucan?: UCANCapMap
+}
+export interface AppAgentForWorker extends AppAgent {
+	signerSecret?: Uint8Array
+}
+export interface SnapRootBlock {
+	applogs: CID
+	applogsSignature: Uint8Array
+	info: CID
+	infoSignature: Uint8Array
+	prev?: CID
+	prevCounter?: number | null  // Sequential counter: 0 for first, then +1 for each. null = unknown (legacy/error).
+}
+export interface SnapBlockLogs {
+	logs: CID[]
+}
+export interface SnapBlockChunks {
+	chunks: CID[]
+}
+export type SnapBlockLogsOrChunks = SnapBlockLogs | SnapBlockChunks
+
+// HACK: this is actually note3 types, not wovin
+export interface IShare {
+	id?: string // string hash of pub (used as unique id in IDB) `W3Name.create().toString()` starts with k51qzi5uqu5d
+	createdAt: string // ISO timestamp of creation
+	name: string // nick name for the pub
+	isDeleted?: boolean
+	purgeBeforePush?: boolean
+
+	pk: Uint8Array // exported privatekey - needed to create WritableName for publishing //TODO: store as non-extractable / encrypted?
+
+	autopush: boolean
+	lastPush: string | null
+	lastCID?: string
+	latestLogTs?: string
+	pubCounter?: number
+
+	publishedBy: string // local user appAgent
+	selectors?: string[] // to be used as a filter for which applogs to pub
+	encryptedFor?: string | null // short agentHash
+	encryptedWith?: CryptoKey | null // AES-GCM derived key from ECDH keys (local private and remote public)
+
+	// HACK WIP #39 - shared encryption
+	sharedKey?: CryptoKey | null // AES-GCM derived key from ECDH keys (local private and ipns public)
+	sharedAgents?: AgentID[] | null // array of string EntityIDs for the chosen agents (we need public jwkd atoms for each of them)
+	sharedKeyMap?: Map<AgentID, string> | null // uses public key from each agent to derive an aes key that is used to encrypt and btoa the sharedKey that is actually used to encrypt and decrypt the applogs
+}
+export interface ISubscription {
+	id: string // string hash of pub (used as unique id in IDB) `W3Name.create().toString()` starts with k51qzi5uqu5d
+	createdAt: string // ISO timestamp of creation
+	name: string // nick name for the pub
+	isDeleted: boolean
+
+	lastPull?: string | null
+	lastPullAttempt?: string | null
+	autopull: boolean
+	realtime?: boolean // enable real-time WebSocket updates via IPNS watcher
+	lastCID?: string // ? why not CidString
+	lastApplogCID?: string
+	publishedBy?: string // remote publisher short agentHash
+	encryptedFor?: string | undefined // short agentHash
+	encryptedWith?: CryptoKey | undefined // AES-GCM derived key from ECDH keys (local private and remote public)
+}
+export function isShare(obj: any): obj is IShare {
+	return obj?.pk !== undefined && obj?.lastPush !== undefined
+}
+export function isSubscription(obj: any): obj is ISubscription {
+	return obj?.lastPull !== undefined
+}
+
+export type TShareSub = IShare | ISubscription
+
+export function agentToShortHash(agentString: string) {
+	return cyrb53hash(agentString, 31, 7) as string
+}

package/src/pubsub/snap-push.ts

@@ -0,0 +1,278 @@
+import * as dagJson from '@ipld/dag-json'
+import { Logger } from 'besonders-logger'
+import { CID } from 'multiformats/cid'
+import stringify from 'safe-stable-stringify'
+import { ensureTsPvAndFinalizeApplog } from '../applog/applog-helpers.ts'
+import type {
+	Applog,
+	ApplogArrayMaybeEncrypted,
+	ApplogArrayMaybeEncryptedRO,
+	ApplogArrayNoCIDMaybeEncryptedRO,
+	ApplogEnc,
+	ApplogEncNoCid,
+	CidString,
+} from '../applog/datom-types.ts'
+import { BlockStoreish, DecodedCar, getDecodedBlock, makeCarBlob } from '../ipfs/car.ts'
+import { encodeBlockOriginal, prepareForPub } from '../ipfs/ipfs-utils.ts'
+import { lastWriteWins } from './../query/basic.ts'
+import { anyOf } from '../query/matchers.ts'
+import { ApplogsOrThread, getLogsFromThread, Thread } from '../thread.ts'
+import { rollingFilter } from '../thread/filters.ts'
+import { keepTruthy } from '../utils.ts'
+import type { AppAgent, IShare, SnapBlockChunks, SnapBlockLogs, SnapBlockLogsOrChunks } from './pubsub-types.ts'
+
+const { WARN, LOG, DEBUG, VERBOSE, ERROR } = Logger.setup(Logger.INFO) // eslint-disable-line no-unused-vars
+
+// export const neverEncryptAttrs = [
+// 	'agent/jwkd',
+// 	'agent/appAgent',
+// 	'pub/encryptedFor',
+// 	'pub/sharedKey',
+// ]
+
+// export interface WovinPublicationInfo {
+// 	id: string
+// }
+
+export async function prepareSnapshotForPush(
+	agent: AppAgent,
+	appThread: Thread,
+	threadToPublish: ApplogsOrThread,
+	share: IShare,
+	prevSnapCID: CID | null,
+	prevCounter: number | null,
+) {
+	if (prevCounter !== null && !prevSnapCID) {
+		throw ERROR('[prepareSnapshotForPush] prevCounter provided without prevSnapCID')
+	}
+	// TODO prevent publish if there is no new info
+	let logsToPublish = getLogsFromThread(threadToPublish)
+
+	// const logsFromLastPush =    await getLogsFromPub(publication)
+	// logsToPublish = logsToPublish.filter(eachLog => !logsFromLastPush.includes(eachLog.cid)) // TODO deep compare includes
+	// const prevPushCIDs = [
+	// 	publication.lastCID,
+	// 	//TODO add this one and update the publication data after push
+	// ]
+	// const includedLogCIDs = [
+	// 	'full array of CIDS from all previous pushes'
+	// ]
+
+	DEBUG(`[preparePubForPush] Collected ${logsToPublish.length} logs :`, {
+		logsToPublish,
+		threadOrLogsCount: (threadToPublish as any).nameAndSizeUntracked || (`[${(threadToPublish as any).length}]`),
+	})
+
+	const { sharedAgents, sharedKeyMap, sharedKey, pubCounter } = share ?? {}
+
+	const getExistingOrNewLog = (thread: Thread, share: IShare, ag: string, at: string, vl) => {
+		let logInQuestion = rollingFilter(lastWriteWins(thread), { en: share.id, at }).latestLog
+		if (!logInQuestion && vl !== undefined) {
+			logInQuestion = ensureTsPvAndFinalizeApplog({ ag, en: share.id, at, vl }, thread)
+		}
+		return logInQuestion // can be undefined if the passed vl is undefined and the log is not found
+	}
+	const shareNameLog = getExistingOrNewLog(appThread, share, agent.ag, 'share/name', share.name)
+
+	// ? using did as it is derived from the same ecdh in note3 and part of the minimal AppAgent type required here in wovin core
+	const shareCounterLog = getExistingOrNewLog(appThread, share, agent.ag, 'share/counter', `${agent.did}<::>${pubCounter}`)
+	// ? discuss if this works to bind the counter to a specific derivation key - did is not necessarily derived from the same key by all lib users
+
+	const encryptApplog = async (applog: Applog, keyToUse: CryptoKey): Promise<Uint8Array> => {
+		const { log: eachLog, cid } = prepareForPub(applog) // without cid
+		const enc = new TextEncoder()
+		const stringified = stringify(eachLog)
+		const stringifiedEncodedAppLogPayload = enc.encode(stringified) // TODO: consider encodeToDagJson instead
+		VERBOSE('[odd]', { eachLog, stringified, stringifiedEncodedAppLogPayload })
+
+		try {
+			// @ts-expect-error
+			const encPayload = await agent.crypto?.aes.encrypt(stringifiedEncodedAppLogPayload, keyToUse, 'AES-GCM')
+			// TODO get rid of odd down here
+			VERBOSE('[odd] encrypted length:', stringifiedEncodedAppLogPayload.length, { encPayload })
+			return encPayload
+		} catch (err) {
+			throw ERROR('FAILED TO ENC payload length:', stringifiedEncodedAppLogPayload.length, { err })
+		}
+
+		// const decrypted = await decryptWithAesSharedKey(encPayload, keyToUse, 'string')
+	}
+
+	let maybeEncryptedApplogs: ApplogEncNoCid[] | readonly Applog[]
+	const encryptedApplogs = [] as { enc: Uint8Array }[]
+	const agentSharedKeyLogs = []
+	if (sharedAgents) { // encrypt all Applogs
+		if (!sharedKey || !sharedKeyMap) {
+			throw ERROR('sharedAgents but no Keys/Map', { sharedAgents, sharedKeyMap, sharedKey })
+		}
+		VERBOSE('encrypting', { sharedAgents, sharedKeyMap })
+
+		for (const [eachAgent, eachEncKey] of Array.from(sharedKeyMap.entries())) {
+			VERBOSE('adding key', { eachAgent, eachEncKey })
+			agentSharedKeyLogs.push({
+				ag: agent.ag,
+				en: eachAgent,
+				at: 'share/sharedKey',
+				vl: eachEncKey, // these are encrypted with the derived key from the local agent private and remote agent public keys
+			})
+		}
+		// const encryptedForLogs = await insertApplogsInAppDB(agentSharedKeyLogs)
+		// DEBUG(`[publish] adding agentSharedKeyLogs:`, encryptedForLogs)
+		const CIDlist: { cid: CidString; encCID?: CidString }[] = []
+		const pubCIDmap: Record<CidString, typeof CIDlist> = {}
+		// TODO ensure that all needed keys are in
+		for (const eachLog of logsToPublish) {
+			VERBOSE('[crypto] encrypting ', { eachLog, sharedKey })
+			// try {
+			const encPayload = await encryptApplog(eachLog, sharedKey)
+			DEBUG('[crypto] encrypted ', { eachLog, encPayload, sharedKey })
+			// } catch (err) {
+			// 	// its already traced in encryptAndTestDecrypt
+			// 	// continue
+			// }
+			encryptedApplogs.push({ enc: encPayload })
+		}
+		maybeEncryptedApplogs = encryptedApplogs
+	} else {
+		maybeEncryptedApplogs = logsToPublish // publish nonEncrypted
+	}
+
+	DEBUG('adding all agent info and shareAtoms', {
+		share,
+		agent,
+		logsToPublish,
+		// threadToPublish, - very verbose
+		agentSharedKeyLogs,
+	})
+	const infoLogs = [
+		...rollingFilter(lastWriteWins(appThread), { // TODO: use static filter for performance
+			en: agent.ag,
+			at: anyOf('agent/ecdh', 'agent/jwkd', 'agent/appAgent'),
+		}).applogs,
+		...(shareNameLog ? [shareNameLog] : []),
+		...(shareCounterLog ? [shareCounterLog] : []),
+		...agentSharedKeyLogs,
+	]
+	DEBUG(`[prepareSnapshotForPush] info logs:`, infoLogs)
+	if (!infoLogs.find(({ at }) => at === 'agent/appAgent')) throw ERROR(`[prepareSnapshotForPush] appThread missing agent/appAgent log`)
+
+	const applogsToEncode = keepTruthy(maybeEncryptedApplogs)
+	const infologsToEncode = keepTruthy(infoLogs)
+	if (!applogsToEncode.length) {
+		throw ERROR('no valid applogs', { agent, maybeEncryptedApplogs, infoLogs, applogsToEncode, infologsToEncode, prevSnapCID })
+	}
+	if (!infologsToEncode.length) {
+		throw ERROR('no valid infologs', { agent, maybeEncryptedApplogs, infoLogs, applogsToEncode, infologsToEncode, prevSnapCID })
+	}
+	const encodedSnapshot = await encodeSnapshotAsCar(agent, applogsToEncode, infologsToEncode, prevSnapCID, prevCounter)
+	DEBUG('inPrepareSnapshotForPush', { encodedSnapshot })
+	return encodedSnapshot
+}
+
+/**
+ * @param applogs Encrypted or plain applogs
+ * @returns Car file
+ */
+export async function encodeSnapshotAsCar(
+	agent: AppAgent,
+	applogs: ApplogArrayNoCIDMaybeEncryptedRO,
+	infoLogs: readonly Applog[],
+	prevSnapCID: CID | null,
+	prevCounter: number | null,
+) {
+	DEBUG(`[encodeSnapshotAsCar] encoding`, { agent, applogs, infoLogs })
+	const { cids: infoLogCids, encodedApplogs: encodedInfoLogs } = await encodeApplogsAsIPLD(infoLogs)
+	const { cids: applogCids, encodedApplogs } = await encodeApplogsAsIPLD(applogs)
+	let blocks = encodedApplogs.concat(encodedInfoLogs)
+	// We need to wrap the array to get a CID
+	const infoLogsWrap = await encodeBlockOriginal({ logs: infoLogCids })
+	blocks.push(infoLogsWrap)
+	const { rootCID: chunkRootCID, blocks: chunkBlocks } = await chunkApplogs(applogCids)
+	blocks = blocks.concat(chunkBlocks) // (i) concat bc. https://stackoverflow.com/a/51860949
+	const infoSignature = await agent.sign(infoLogsWrap.cid.bytes)
+	const applogsSignature = await agent.sign(chunkRootCID.bytes)
+	const root = {
+		info: infoLogsWrap.cid,
+		applogs: chunkRootCID,
+		infoSignature,
+		applogsSignature,
+		prev: prevSnapCID,
+		prevCounter: !prevSnapCID ? 0 : prevCounter !== null ? prevCounter + 1 : null,
+	}
+	DEBUG('[encodeSnapshotAsCar] encoding root', { root, logCids: applogCids, infoLogCids })
+	const encodedRoot = await encodeBlockOriginal(root)
+	blocks.push(encodedRoot)
+	DEBUG('[encodeSnapshotAsCar] => root', { encodedRoot })
+
+	return {
+		cid: encodedRoot.cid,
+		blob: await makeCarBlob(encodedRoot.cid, blocks), // TODO: create CarBuilder (incl .encodeAndAdd({...}))
+		blocks,
+		infoLogCids,
+		applogCids,
+	}
+}
+
+/** (i) IPFS has a block size limit of 1MB - which is about 15K CIDs */
+export async function chunkApplogs(applogCids: CID<unknown, 297, 18, 1>[], size = 10000) {
+	if (!applogCids.length) throw ERROR(`[chunkApplogs] called with empty array`)
+	const chunks = []
+	// TODO: chunk by stable btree based on size or something like that
+	for (let i = 0; i < applogCids.length; i += size) {
+		const chunk = await encodeBlockOriginal({ logs: applogCids.slice(i, Math.min(i + applogCids.length, i + size)) })
+		chunks.push(chunk)
+	}
+	if (chunks.length === 1) return { rootCID: chunks[0].cid, blocks: chunks }
+	const root = await encodeBlockOriginal({ chunks: chunks.map(chunk => chunk.cid) })
+	const blocks = [root, ...chunks]
+	DEBUG(`[chunkApplogs] ${applogCids.length} logs chunked into ${chunks.length}`, { applogCids, root, blocks, chunks, dagJson })
+	return { rootCID: root.cid, blocks, chunks }
+}
+export async function unchunkApplogsBlock(block: SnapBlockLogsOrChunks, blockStore: BlockStoreish): Promise<CID[]> {
+	if (isSnapBlockChunks(block)) {
+		return (await Promise.all(
+			block.chunks.map(async (chunkCid) => {
+				const block = (await getDecodedBlock(blockStore, chunkCid)) as SnapBlockLogs
+				if (!block.logs) throw ERROR(`Weird chunk`, block)
+				return block.logs
+			}),
+		)).flat()
+	} else {
+		return block.logs
+	}
+}
+export function isSnapBlockChunks(block: SnapBlockLogsOrChunks): block is SnapBlockChunks {
+	return (block as any).chunks
+}
+/**
+ * @param applogs Encrypted or plain applogs
+ * @returns Car file
+ */
+export async function encodeSnapshotApplogsAsCar(
+	applogs: ApplogArrayMaybeEncryptedRO,
+) {
+	const encoded = await encodeApplogsAsIPLD(applogs)
+	if (!encoded) throw ERROR('invalid applogs cannot continue', { applogs, encoded })
+	const { cids, encodedApplogs } = encoded
+	const root = { applogs: cids }
+	const encodedRoot = await encodeBlockOriginal(root)
+	DEBUG('[encodeSnapshotApplogsAsCar] encoded root', { cids, encodedRoot })
+
+	return await makeCarBlob(encodedRoot.cid, [encodedRoot, ...encodedApplogs])
+}
+
+async function encodeApplogsAsIPLD(applogs: ApplogArrayNoCIDMaybeEncryptedRO) {
+	DEBUG({ applogs })
+	const validApplogs = applogs.filter(eachLog => !!eachLog)
+	DEBUG({ validApplogs })
+	if (!validApplogs.length) throw ERROR('no valid applogs')
+	const preppedLogs = validApplogs.map(log => prepareForPub(log as Applog).log)
+	const encodedApplogs = await Promise.all(preppedLogs.map(encodeBlockOriginal))
+	DEBUG('[encodeApplogsAsIpld] encoded applogs', { preppedLogs, encodedApplogs })
+
+	const cids = encodedApplogs.map(b => {
+		if (!b.cid) throw ERROR(`[publish] no cid for encoded log:`, b)
+		return b.cid
+	})
+	return { cids, encodedApplogs }
+}

package/src/pubsub/ucan-example.ts

@@ -0,0 +1,61 @@
+import { Capability } from 'iso-ucan/capability'
+import { Store } from 'iso-ucan/store'
+import { MemoryDriver } from 'iso-kv/drivers/memory.js'
+import { EdDSASigner } from 'iso-signatures/signers/eddsa.js'
+import { z } from 'zod'
+import { verify } from 'iso-signatures/verifiers/eddsa.js'
+import { Resolver } from 'iso-signatures/verifiers/resolver.js'
+
+export const verifierResolver = new Resolver({
+  Ed25519: verify,
+})
+// from:
+//https://ucan.xyz/guides/examples/
+
+// Initialize delegation store for tracking capability chains
+// In production, this might be backed by a database or persistent storage
+const store = new Store(new MemoryDriver())
+
+// Define file read capability with path validation schema
+// The schema ensures all invocations include a valid file path
+const FileReadCap = Capability.from({
+  schema: z.object({
+    path: z.string(),  // Required: file path to read
+  }),
+  cmd: '/file/read',   // UCAN v1 command identifier
+	verifierResolver,
+})
+
+// Create cryptographic identities for resource owner and accessor
+// In a real system, these would be persistent identity keypairs
+const alice = await EdDSASigner.generate()  // Resource owner
+const bob = await EdDSASigner.generate()    // Requesting access
+
+const nowInSeconds = Math.floor(Date.now() / 1000)
+
+// Alice grants Bob permission to read files
+// This delegation can be stored, transmitted, or embedded in applications
+const delegation = await FileReadCap.delegate({
+  iss: alice,    // Alice issues this capability
+  aud: bob,      // Bob is authorized to use it
+  sub: alice,    // Alice's resources are the subject
+  pol: [],      // No additional policy constraints
+  exp: nowInSeconds + 3600, // Expires in 1 hour for security
+  store,
+})
+
+// Store delegation to enable later invocation validation
+// The store enables automatic delegation chain resolution
+// await store.set(delegation)
+
+// Bob exercises the delegated capability to read a specific file
+// This creates a cryptographically verifiable access request
+const invocation = await FileReadCap.invoke({
+  iss: bob,      // Bob is invoking the capability
+  sub: alice,    // Alice's system will process the request
+  args: {
+    path: '/documents/report.pdf'  // Specific file Bob wants to read
+  },
+  store,         // Store containing the delegation proof
+  exp: nowInSeconds + 300, // Invocation expires in 5 minutes
+})

package/src/pubsub/ucan.ts

@@ -0,0 +1,56 @@
+import type { DelegationOptions, DelegationPayload, PayloadBase, StandardSchemaV1 } from 'iso-ucan/types'
+import { Delegation } from 'iso-ucan/delegation'
+import { AgentID } from '../applog/datom-types.ts'
+import { Capability } from 'iso-ucan/capability'
+import { z } from 'zod/mini'
+import { verify } from 'iso-signatures/verifiers/eddsa.js'
+import { Resolver } from 'iso-signatures/verifiers/resolver.js'
+
+export const verifierResolver = new Resolver({
+  Ed25519: verify,// TODO review if iso-signatures is better than web crypto (or if it even uses it)
+})
+
+
+const WovinRootCap = Capability.from({
+	cmd: '/wovin',
+	schema: z.object({
+		agentString: z.string(),
+		agentID: z.string(),
+    	type: z.string(),
+	}),
+	verifierResolver,
+})
+
+const WovinPublishCap = Capability.from({
+	cmd: '/wovin/publish',
+	schema: z.object({
+		agentString: z.string(),
+		agentID: z.string(),
+    	type: z.string(),
+	}),
+	verifierResolver,
+})
+const WovinDelegateCap = Capability.from({
+	cmd: '/wovin/delegate',
+	schema: z.object({
+		agentString: z.string(),
+		agentID: z.string(),
+    	type: z.string(),
+	}),
+		verifierResolver,
+})
+export enum KnownCaps {
+	publish = 'WovinPublishCap',
+	delegate = 'cap_delegate',
+}
+
+type DelegationPayloadSchema = StandardSchemaV1<unknown, DelegationPayload<unknown>>
+export type DelegationOptionsStandard = DelegationOptions<DelegationPayloadSchema>
+export type UCANCapMap = Partial<Record<KnownCaps, DelegationOptionsStandard[]>>
+
+export type AgentCapMap = Record<AgentID, UCANCapMap>
+
+export const createDelegation = (delegationPayload:DelegationOptionsStandard) => {
+	// delegationPayload.
+	return  Delegation.create(delegationPayload)
+}

package/src/pubsub.ts

@@ -0,0 +1,4 @@
+export * from './pubsub/connector.ts'
+export * from './pubsub/pub-pull.ts'
+export * from './pubsub/snap-push.ts'
+export * from './pubsub/pubsub-types.ts'

package/src/query/attr-helpers.ts

@@ -0,0 +1,5 @@
+/** Strip everything up to and including the first `/` */
+export type StripFirstPrefix<S extends string> = S extends `${string}/${infer Rest}` ? Rest : S
+
+/** Strip a specific prefix `P/` from attribute string */
+export type StripExplicitPrefix<S extends string, P extends string> = S extends `${P}/${infer Rest}` ? Rest : S