@livestore/common 0.0.58-dev.5 → 0.0.58-dev.7

package/src/sync/next/compact-events.ts

@@ -0,0 +1,219 @@
+import { graphologyDag } from '@overtone/utils'
+
+import { replacesFacts } from './facts.js'
+import type { HistoryDag } from './history-dag.js'
+import { emptyHistoryDag, eventIdToString } from './history-dag.js'
+
+/**
+ * Idea:
+ * - iterate over all events from leaves to root
+ * - for each event
+ *   - gradually make sub dags by following the event's fact dependencies
+ *   - for each sub dag check and remove sub dags further up in the history dag that are a subset of the current sub dag
+ *
+ * TODO: try to implement this function on top of SQLite
+ */
+export const compactEvents = (inputDag: HistoryDag): { dag: HistoryDag; compactedEventCount: number } => {
+  const dag = inputDag.copy()
+  const compactedEventCount = 0
+
+  const orderedEventIdStrs = graphologyDag.topologicalSort(dag).reverse()
+
+  // drop root
+  orderedEventIdStrs.pop()
+
+  for (const eventIdStr of orderedEventIdStrs) {
+    if (dag.hasNode(eventIdStr) === false) {
+      continue
+    }
+
+    const subDagsForEvent = Array.from(makeSubDagsForEvent(dag, eventIdStr))
+    for (const subDag of subDagsForEvent) {
+      let shouldRetry = true
+      while (shouldRetry) {
+        const subDagsInHistory = findSubDagsInHistory(dag, subDag, eventIdStr)
+
+        // console.debug(
+        //   'subDagsInHistory',
+        //   eventIdStr,
+        //   'target',
+        //   subDag.nodes(),
+        //   'found',
+        //   ...subDagsInHistory.subDags.map((_) => _.nodes()),
+        // )
+
+        for (const subDagInHistory of subDagsInHistory.subDags) {
+          if (dagDependsOnDag(subDag, subDagInHistory, dag) === false) {
+            dropFromDag(dag, subDagInHistory)
+          }
+        }
+
+        // Sometimes some sub dags are ommitted because they depended on other sub dags in same batch.
+        // We can retry to also remove those.
+        // Implementation: retry if outsideDependencies overlap with deleted sub dags
+        if (
+          subDagsInHistory.allOutsideDependencies.some((outsideDependencies) =>
+            outsideDependencies.every((dep) => subDagsInHistory.subDags.some((subDag) => subDag.hasNode(dep))),
+          ) === false
+        ) {
+          shouldRetry = false
+        }
+      }
+    }
+  }
+
+  return { dag, compactedEventCount }
+}
+
+function* makeSubDagsForEvent(inputDag: HistoryDag, eventIdStr: string): Generator<HistoryDag> {
+  /** Map from eventIdStr to array of eventIdStrs that are dependencies */
+  let nextIterationEls: Map<string, string[]> = new Map([[eventIdStr, []]])
+  let previousDag: HistoryDag | undefined
+
+  while (nextIterationEls.size > 0) {
+    // start with a copy of the last sub dag to build on top of
+    const subDag = previousDag?.copy() ?? emptyHistoryDag()
+
+    const currentIterationEls = new Map(nextIterationEls)
+    nextIterationEls = new Map()
+
+    for (const [currentEventIdStr, edgeTargetIdStrs] of currentIterationEls) {
+      const node = inputDag.getNodeAttributes(currentEventIdStr)
+      if (subDag.hasNode(currentEventIdStr) === false) {
+        subDag.addNode(currentEventIdStr, { ...node })
+      }
+      for (const edgeTargetIdStr of edgeTargetIdStrs) {
+        subDag.addEdge(currentEventIdStr, edgeTargetIdStr, { type: 'facts' })
+      }
+
+      for (const depEdge of inputDag.outboundEdgeEntries(currentEventIdStr)) {
+        if (depEdge.attributes.type === 'facts') {
+          const depEventIdStr = depEdge.target
+          nextIterationEls.set(depEventIdStr, [...(nextIterationEls.get(depEventIdStr) ?? []), currentEventIdStr])
+        }
+      }
+    }
+
+    previousDag = subDag
+
+    // console.debug('subDag yield', subDag.nodes())
+    yield subDag
+  }
+}
+
+/**
+ * Iterates over all events from root to `upToExclEventIdStr`
+ * and collects all valid sub dags that are replaced by `targetSubDag`.
+ */
+const findSubDagsInHistory = (
+  inputDag: HistoryDag,
+  targetSubDag: HistoryDag,
+  upToExclEventIdStr: string,
+): { subDags: HistoryDag[]; allOutsideDependencies: string[][] } => {
+  const subDags: HistoryDag[] = []
+  const allOutsideDependencies: string[][] = []
+
+  for (const eventIdStr of graphologyDag.topologicalSort(inputDag)) {
+    if (eventIdStr === upToExclEventIdStr) {
+      break
+    }
+
+    for (const subDag of makeSubDagsForEvent(inputDag, eventIdStr)) {
+      // console.debug('findSubDagsInHistory', 'target', targetSubDag.nodes(), 'subDag', subDag.nodes())
+      if (subDag.size < targetSubDag.size) {
+        continue
+      }
+
+      const outsideDependencies = outsideDependenciesForDag(subDag, inputDag)
+      if (outsideDependencies.length > 0) {
+        allOutsideDependencies.push(outsideDependencies)
+      }
+
+      if (outsideDependencies.length === 0 && dagReplacesDag(subDag, targetSubDag)) {
+        subDags.push(subDag)
+      } else {
+        break
+      }
+    }
+  }
+
+  return { subDags, allOutsideDependencies }
+}
+
+const dropFromDag = (dag: HistoryDag, subDag: HistoryDag) => {
+  for (const nodeIdStr of subDag.nodes()) {
+    removeEvent(dag, nodeIdStr)
+  }
+}
+
+/** Returns outside dependencies of `subDag` (but inside `inputDag`) */
+const outsideDependenciesForDag = (subDag: HistoryDag, inputDag: HistoryDag) => {
+  const outsideDependencies = []
+  for (const nodeIdStr of subDag.nodes()) {
+    for (const edgeEntry of inputDag.outboundEdgeEntries(nodeIdStr)) {
+      if (edgeEntry.attributes.type === 'facts') {
+        const depEventIdStr = edgeEntry.target
+        if (subDag.hasNode(depEventIdStr) === false) {
+          outsideDependencies.push(depEventIdStr)
+        }
+      }
+    }
+  }
+
+  return outsideDependencies
+}
+
+/** Checks whether dagA depends on dagB */
+const dagDependsOnDag = (dagA: HistoryDag, dagB: HistoryDag, inputDag: HistoryDag): boolean => {
+  for (const nodeAIdStr of dagA.nodes()) {
+    for (const edgeEntryA of inputDag.inboundEdgeEntries(nodeAIdStr)) {
+      if (edgeEntryA.attributes.type === 'facts') {
+        const depNodeIdStr = edgeEntryA.target
+        if (dagB.hasNode(depNodeIdStr)) {
+          return true
+        }
+      }
+    }
+  }
+
+  return false
+}
+
+/** Checks if dagA replaces dagB */
+const dagReplacesDag = (dagA: HistoryDag, dagB: HistoryDag): boolean => {
+  if (dagA.size !== dagB.size) {
+    return false
+  }
+
+  // TODO write tests that covers deterministic order when DAGs have branches
+  const nodeEntriesA = graphologyDag.topologicalSort(dagA).map((nodeId) => dagA.getNodeAttributes(nodeId))
+  const nodeEntriesB = graphologyDag.topologicalSort(dagB).map((nodeId) => dagB.getNodeAttributes(nodeId))
+
+  for (let i = 0; i < nodeEntriesA.length; i++) {
+    const nodeA = nodeEntriesA[i]!
+    const nodeB = nodeEntriesB[i]!
+
+    if (replacesFacts(nodeA.factsGroup, nodeB.factsGroup) === false) {
+      return false
+    }
+  }
+
+  return true
+}
+
+const removeEvent = (dag: HistoryDag, eventIdStr: string) => {
+  // console.debug('removing event', eventIdStr)
+  const event = dag.getNodeAttributes(eventIdStr)
+  const parentIdStr = eventIdToString(event.parentId)
+  const childEdges = dag.outboundEdgeEntries(eventIdStr)
+
+  for (const childEdge of childEdges) {
+    if (childEdge.attributes.type === 'parent') {
+      const childEvent = dag.getNodeAttributes(childEdge.target)
+      childEvent.parentId = { ...event.parentId }
+      dag.addEdge(parentIdStr, eventIdToString(childEvent.id), { type: 'parent' })
+    }
+  }
+
+  dag.dropNode(eventIdStr)
+}

package/src/sync/next/facts.ts

@@ -0,0 +1,228 @@
+import { graphologyDag, notYetImplemented } from '@overtone/utils'
+
+import type { EventId } from '../../adapter-types.js'
+import type {
+  FactsCallback,
+  MutationEventFactInput,
+  MutationEventFacts,
+  MutationEventFactsGroup,
+  MutationEventFactsSnapshot,
+} from '../../schema/mutations.js'
+import { EMPTY_FACT_VALUE, type HistoryDag, type HistoryDagNode } from './history-dag.js'
+
+export const factsSnapshotForEvents = (events: HistoryDagNode[], endEventId: EventId): MutationEventFactsSnapshot => {
+  const facts = new Map<string, any>()
+
+  for (const event of events) {
+    if (compareEventIds(event.id, endEventId) > 0) {
+      return facts
+    }
+
+    applyFactGroup(event.factsGroup, facts)
+  }
+
+  return facts
+}
+
+export const factsSnapshotForDag = (dag: HistoryDag, endEventId: EventId | undefined): MutationEventFactsSnapshot => {
+  const facts = new Map<string, any>()
+
+  const orderedEventIdStrs = graphologyDag.topologicalSort(dag)
+
+  for (let i = 0; i < orderedEventIdStrs.length; i++) {
+    const event = dag.getNodeAttributes(orderedEventIdStrs[i]!)
+    if (endEventId !== undefined && compareEventIds(event.id, endEventId) > 0) {
+      return facts
+    }
+
+    applyFactGroup(event.factsGroup, facts)
+  }
+
+  return facts
+}
+
+export type FactValidationResult =
+  | {
+      success: true
+    }
+  | {
+      success: false
+      /** Index of the item that caused the validation to fail */
+      index: number
+      requiredFacts: MutationEventFacts
+      mismatch: {
+        existing: MutationEventFacts
+        required: MutationEventFacts
+      }
+      currentSnapshot: MutationEventFacts
+    }
+
+export const validateFacts = ({
+  factGroups,
+  initialSnapshot,
+}: {
+  factGroups: MutationEventFactsGroup[]
+  initialSnapshot: MutationEventFactsSnapshot
+}): FactValidationResult => {
+  const currentSnapshot = new Map(initialSnapshot)
+
+  for (const [index, factGroup] of factGroups.entries()) {
+    if (isSubSetMapByValue(factGroup.depRequire, currentSnapshot) === false) {
+      const existing = new Map()
+      const required = new Map()
+
+      for (const [key, value] of factGroup.depRequire) {
+        if (currentSnapshot.get(key) !== value) {
+          existing.set(key, currentSnapshot.get(key))
+          required.set(key, value)
+        }
+      }
+
+      return {
+        success: false,
+        index,
+        requiredFacts: factGroup.depRequire,
+        currentSnapshot,
+        mismatch: { existing, required },
+      }
+    }
+
+    applyFactGroup(factGroup, currentSnapshot)
+  }
+
+  return {
+    success: true,
+  }
+}
+
+export const applyFactGroups = (factGroups: MutationEventFactsGroup[], snapshot: MutationEventFactsSnapshot) => {
+  for (const factGroup of factGroups) {
+    applyFactGroup(factGroup, snapshot)
+  }
+}
+
+export const applyFactGroup = (factGroup: MutationEventFactsGroup, snapshot: MutationEventFactsSnapshot) => {
+  for (const [key, value] of factGroup.modifySet) {
+    snapshot.set(key, value)
+  }
+
+  for (const [key, _value] of factGroup.modifyUnset) {
+    snapshot.delete(key)
+  }
+}
+
+/** Check if setA is a subset of setB */
+const isSubSetMapByValue = (setA: MutationEventFacts, setB: MutationEventFacts) => {
+  for (const [key, value] of setA) {
+    if (setB.get(key) !== value) {
+      return false
+    }
+  }
+  return true
+}
+
+/** Check if setA is a subset of setB */
+const isSubSetMapByKey = (setA: MutationEventFacts, setB: MutationEventFacts) => {
+  for (const [key, _value] of setA) {
+    if (!setB.has(key)) {
+      return false
+    }
+  }
+  return true
+}
+
+/** Check if groupA depends on groupB */
+export const dependsOn = (groupA: MutationEventFactsGroup, groupB: MutationEventFactsGroup): boolean =>
+  factsIntersect(groupA.depRead, groupB.modifySet) ||
+  factsIntersect(groupA.depRead, groupB.modifyUnset) ||
+  factsIntersect(groupA.depRequire, groupB.modifySet) ||
+  factsIntersect(groupA.depRequire, groupB.modifyUnset)
+
+export const replacesFacts = (groupA: MutationEventFactsGroup, groupB: MutationEventFactsGroup): boolean => {
+  const replaces = (a: MutationEventFacts, b: MutationEventFacts) => a.size > 0 && b.size > 0 && isSameMapByKey(a, b)
+
+  const noFactsOrSame = (a: MutationEventFacts, b: MutationEventFacts) =>
+    a.size === 0 || b.size === 0 || isSameMapByKey(a, b)
+
+  return (
+    (replaces(groupA.modifySet, groupB.modifySet) && noFactsOrSame(groupA.modifyUnset, groupB.modifyUnset)) ||
+    (replaces(groupA.modifySet, groupB.modifyUnset) && noFactsOrSame(groupA.modifyUnset, groupB.modifySet)) ||
+    (replaces(groupA.modifyUnset, groupB.modifySet) && noFactsOrSame(groupA.modifySet, groupB.modifyUnset)) ||
+    (replaces(groupA.modifyUnset, groupB.modifyUnset) && noFactsOrSame(groupA.modifySet, groupB.modifySet))
+  )
+}
+
+export const isSameMapByKey = (set: MutationEventFacts, otherSet: MutationEventFacts) =>
+  set.size === otherSet.size && isSubSetMapByKey(set, otherSet)
+
+export const factsToString = (facts: MutationEventFacts) => {
+  return Array.from(facts)
+    .map(([key, value]) => (value === EMPTY_FACT_VALUE ? key : `${key}=${value}`))
+    .join(', ')
+}
+
+export const factsIntersect = (setA: MutationEventFacts, setB: MutationEventFacts): boolean => {
+  for (const [key, _value] of setA) {
+    if (setB.has(key)) {
+      return true
+    }
+  }
+  return false
+}
+
+export const getFactsGroupForMutationArgs = ({
+  factsCallback,
+  args,
+  currentFacts,
+}: {
+  factsCallback: FactsCallback<any> | undefined
+  args: any
+  currentFacts: MutationEventFactsSnapshot
+}): MutationEventFactsGroup => {
+  const depRead: MutationEventFactsSnapshot = new Map<string, any>()
+  const factsSnapshotProxy = new Proxy(currentFacts, {
+    get: (target, prop) => {
+      if (prop === 'has') {
+        return (key: string) => {
+          depRead.set(key, EMPTY_FACT_VALUE)
+          return target.has(key)
+        }
+      } else if (prop === 'get') {
+        return (key: string) => {
+          depRead.set(key, EMPTY_FACT_VALUE)
+          return target.get(key)
+        }
+      }
+
+      notYetImplemented(`getFactsGroupForMutationArgs: ${prop.toString()} is not yet implemented`)
+    },
+  })
+
+  const factsRes = factsCallback?.(args, factsSnapshotProxy)
+  const iterableToMap = (iterable: Iterable<MutationEventFactInput>) => {
+    const map = new Map()
+    for (const item of iterable) {
+      if (typeof item === 'string') {
+        map.set(item, EMPTY_FACT_VALUE)
+      } else {
+        map.set(item[0], item[1])
+      }
+    }
+    return map
+  }
+  const facts = {
+    modifySet: factsRes?.modify.set ? iterableToMap(factsRes.modify.set) : new Map(),
+    modifyUnset: factsRes?.modify.unset ? iterableToMap(factsRes.modify.unset) : new Map(),
+    depRequire: factsRes?.require ? iterableToMap(factsRes.require) : new Map(),
+    depRead,
+  }
+
+  return facts
+}
+
+export const compareEventIds = (a: EventId, b: EventId) => {
+  if (a.global !== b.global) {
+    return a.global - b.global
+  }
+  return a.local - b.local
+}

package/src/sync/next/graphology.ts

@@ -0,0 +1,49 @@
+// TODO re-enable when `graphology` supports ESM `exports` and `.js` import/export syntax
+// import {} from 'graphology'
+import * as graphology_ from 'graphology'
+import type * as graphologyTypes from 'graphology-types'
+
+export const graphology = graphology_ as any
+
+export declare class IGraph<
+  NodeAttributes extends graphologyTypes.Attributes = graphologyTypes.Attributes,
+  EdgeAttributes extends graphologyTypes.Attributes = graphologyTypes.Attributes,
+  GraphAttributes extends graphologyTypes.Attributes = graphologyTypes.Attributes,
+> extends graphologyTypes.AbstractGraph<NodeAttributes, EdgeAttributes, GraphAttributes> {
+  constructor(options?: graphologyTypes.GraphOptions)
+}
+
+export const DirectedGraph = class DirectedGraph extends graphology.DirectedGraph {
+  constructor(options?: graphologyTypes.GraphOptions) {
+    super(options)
+  }
+} as typeof IGraph
+
+export const Graph = class Graph extends graphology.Graph {
+  constructor(options?: graphologyTypes.GraphOptions) {
+    super(options)
+  }
+} as typeof IGraph
+
+// export const graphology = graphology_ as graphologyTypes
+
+/*
+
+Example usage:
+
+const dag = new graphology.DirectedGraph({ allowSelfLoops: false })
+
+nodes.forEach((node) => dag.addNode(node.id, { width: node.data.label.length * 100, height: 40 }))
+edges.forEach((edge) => {
+	// TODO do this filtering earlier
+	if (!nodeIds.has(edge.source) || !nodeIds.has(edge.target)) return
+
+	dag.addEdge(edge.source, edge.target)
+})
+
+graphologyLayout.random.assign(dag) // needed for initial `x`, `y` values
+const sensibleSettings = forceAtlas2.inferSettings(dag)
+// forceAtlas2.assign(dag, { iterations: 100, settings: { adjustSizes: true,  } })
+forceAtlas2.assign(dag, { iterations: 100, settings: sensibleSettings })
+
+*/

package/src/sync/next/graphology_.ts

@@ -0,0 +1,2 @@
+export * as graphology from './graphology.js'
+export { default as graphologyDag } from 'graphology-dag'

package/src/sync/next/history-dag.ts

@@ -0,0 +1,109 @@
+import { type EventId, ROOT_ID } from '../../adapter-types.js'
+import type { MutationEventFactsGroup } from '../../schema/mutations.js'
+import { factsToString, validateFacts } from './facts.js'
+import { graphology } from './graphology_.js'
+
+export const connectionTypeOptions = ['parent', 'facts'] as const
+export type ConnectionType = (typeof connectionTypeOptions)[number]
+
+/**
+ * Eventlog represented as a multi-DAG including edges for
+ * - total-order (parent) relationships
+ * - dependency (requires/reads facts) relationships
+ */
+export type HistoryDag = graphology.IGraph<HistoryDagNode, { type: ConnectionType }>
+
+export const emptyHistoryDag = (): HistoryDag =>
+  new graphology.Graph({
+    allowSelfLoops: false,
+    multi: true,
+    type: 'directed',
+  })
+
+// TODO consider making `ROOT_ID` parent to itself
+const rootParentId = { global: ROOT_ID.global - 1, local: 0 } satisfies EventId
+
+export type HistoryDagNode = {
+  id: EventId
+  parentId: EventId
+  mutation: string
+  args: any
+  /** Facts are being used for conflict detection and history compaction */
+  factsGroup: MutationEventFactsGroup
+  meta?: any
+}
+
+export const rootEventNode: HistoryDagNode = {
+  id: ROOT_ID,
+  parentId: rootParentId,
+  // unused below
+  mutation: '__Root__',
+  args: {},
+  factsGroup: { modifySet: new Map(), modifyUnset: new Map(), depRequire: new Map(), depRead: new Map() },
+}
+
+export const EMPTY_FACT_VALUE = Symbol('EMPTY_FACT_VALUE')
+
+export const eventIdToString = (eventId: EventId) =>
+  eventId.local === 0 ? eventId.global.toString() : `${eventId.global}.${eventId.local}`
+
+export const historyDagFromNodes = (dagNodes: HistoryDagNode[], options?: { skipFactsCheck: boolean }) => {
+  if (options?.skipFactsCheck !== true) {
+    const validationResult = validateFacts({
+      factGroups: dagNodes.map((node) => node.factsGroup),
+      initialSnapshot: new Map<string, any>(),
+    })
+
+    if (validationResult.success === false) {
+      throw new Error(
+        `Mutation ${dagNodes[validationResult.index]!.mutation} requires facts that have not been set yet.\nRequires: ${factsToString(validationResult.requiredFacts)}\nFacts Snapshot: ${factsToString(validationResult.currentSnapshot)}`,
+      )
+    }
+  }
+
+  const dag = emptyHistoryDag()
+
+  dagNodes.forEach((node) => dag.addNode(eventIdToString(node.id), node))
+
+  dagNodes.forEach((node) => {
+    if (eventIdToString(node.parentId) !== eventIdToString(rootParentId)) {
+      dag.addEdge(eventIdToString(node.parentId), eventIdToString(node.id), { type: 'parent' })
+    }
+  })
+
+  dagNodes.forEach((node) => {
+    const factKeys = [...node.factsGroup.depRequire.keys(), ...node.factsGroup.depRead.keys()]
+    for (const factKey of factKeys) {
+      // Find the first ancestor node with a matching fact key (via modifySet or modifyUnset) by traversing the graph backwards via the parent edges
+      const depNode = (() => {
+        let currentIdStr = eventIdToString(node.id)
+
+        while (currentIdStr !== eventIdToString(rootParentId)) {
+          const parentEdge = dag.inEdges(currentIdStr).find((e) => dag.getEdgeAttribute(e, 'type') === 'parent')
+          if (!parentEdge) return null
+
+          const parentIdStr = dag.source(parentEdge)
+          const parentNode = dag.getNodeAttributes(parentIdStr)
+
+          if (parentNode.factsGroup.modifySet.has(factKey) || parentNode.factsGroup.modifyUnset.has(factKey)) {
+            return parentNode
+          }
+
+          currentIdStr = parentIdStr
+        }
+
+        return null
+      })()
+
+      if (depNode) {
+        const depNodeIdStr = eventIdToString(depNode.id)
+        const nodeIdStr = eventIdToString(node.id)
+        if (dag.edges(depNodeIdStr, nodeIdStr).filter((e) => dag.getEdgeAttributes(e).type === 'facts').length === 0) {
+          dag.addEdge(depNodeIdStr, nodeIdStr, { type: 'facts' })
+        }
+      }
+    }
+  })
+
+  return dag
+}

package/src/sync/next/mod.ts

@@ -0,0 +1,4 @@
+export * from './history-dag.js'
+export * from './facts.js'
+export * from './compact-events.js'
+export * from './rebase-events.js'