@live-change/simple-query 0.9.156 → 0.9.158

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@live-change/simple-query",
-  "version": "0.9.156",
+  "version": "0.9.158",
   "description": "",
   "main": "src/index.js",
   "scripts": {
@@ -22,7 +22,7 @@
   },
   "type": "module",
   "dependencies": {
-    "@live-change/framework": "^0.9.156",
+    "@live-change/framework": "^0.9.158",
     "pluralize": "^8.0.0"
   },
   "devDependencies": {
@@ -30,5 +30,5 @@
     "typedoc-plugin-markdown": "^4.6.3",
     "typedoc-plugin-rename-defaults": "^0.7.3"
   },
-  "gitHead": "3042fdd1df2a3b696fb7f161304eb23d71b70ad5"
+  "gitHead": "bd30de9030c65b93fbb26b6879f574d5f93baea3"
 }

package/src/query.ts

@@ -6,6 +6,16 @@ import { ModelDefinition, ForeignModelDefinition } from "@live-change/framework"
 
 import { PropertyDefinition } from "@live-change/framework"
 
+import { fileURLToPath } from 'url'
+import { dirname, join, resolve } from 'path'
+import { accessSync, readFileSync } from 'fs'
+
+const simpleIndexCodePath = resolve(dirname(fileURLToPath(import.meta.url)), 'autoIndex.db.js')
+const simpleIndexCode = readFileSync(simpleIndexCodePath, 'utf8')
+const simpleQueryCodePath = resolve(dirname(fileURLToPath(import.meta.url)), 'simpleQuery.db.js')
+const simpleQueryCode = readFileSync(simpleQueryCodePath, 'utf8')
+
+
 interface Range {
   gt?: string
   gte?: string
@@ -121,7 +131,7 @@ class IndexInfo {
     if(this.sources.every(source => source === firstSource)) this.singleSource = firstSource    
     if(this.singleSource) {      
       this.name = name || ''
-      + (this.singleSource.input.$source.serviceName === service.name ? "" : this.singleSource.input.$source.serviceName + "_")
+      + (this.singleSource.input.$source.serviceName === service.name ? "" : service.name + "_")
       + (
         this.singleSource.input.$source.getTypeName() + "_" + (indexParts
         .map(part => part.path.join(".")).join("_"))
@@ -225,6 +235,8 @@ export class QueryDefinition<SDS extends ServiceDefinitionSpecification> {
 
     //this.printDependencies() 
 
+   // process.exit(0)
+
   }
 
   printRules() {
@@ -370,10 +382,15 @@ export class QueryDefinition<SDS extends ServiceDefinitionSpecification> {
     /// Poza tym przy pobieraniu wstecz nie koniecznie potrzeba indexów,
     ///  może trzeba wprowadzić index-forward i index-backward ?
     /// A może trzeba wprowadzić analizę tego co mamy i co chcemy uzyskać w przyszłości ?
+
     const next = source.dependentBy.map(dependent => {
       const otherSource = this.ruleSources.find(s => s.rule === dependent.rule && s != dependent)
       return this.computeSourceExecutionPlan(otherSource, [...resultParameters, source.input.$alias])
     })
+
+    console.log("COMPUTING SOURCE EXECUTION PLAN FOR", source.input.$_toQueryDescription(), "WITH", resultParameters)
+    console.log("RULE", queryDescription(source.rule, '  '))
+
     const ruleParameters = JSON.parse(JSON.stringify(source.rule.$_parametersJSON(resultParameters)))
     if(source.index) {                  
       const indexExecution = {
@@ -521,25 +538,87 @@ export class QueryDefinition<SDS extends ServiceDefinitionSpecification> {
     })
   }
 
-  prepareQuery() {
-    console.log("CREATE INDEXES", this.indexes)
+  createIndexes() {
+    //console.log("CREATE INDEXES", this.indexes)
 
-    for(const index of this.indexes) {
+    /* for(const index of this.indexes) {
       const indexQuery = index.$_createQuery(this.service)
       indexQuery.createIndex(index.name, index.indexParts)
+    } */
+
+    const indexesWithQueries = this.indexes.map(index => ({ info: index, query: index.$_createQuery(this.service) }))
+    for(const indexQuery of indexesWithQueries) {
+      //console.log("CREATE INDEX", indexQuery.info.name)
+      //this.printRules()
+      //console.log("OUTPUT MAPPINGS", indexQuery.info.indexParts)
+      indexQuery.query.computeIndexPlan(indexQuery.info.indexParts)
+      //console.log("INDEX", indexQuery.info.name, "PLAN", JSON.stringify(indexQuery.query.indexPlan, null, 2))
     }
 
-    process.exit(0)
+    for(let i = 0; i < indexesWithQueries.length; i++) {
+      const a = indexesWithQueries[i]
+      for(let j = i+1; j < indexesWithQueries.length; j++) {
+        const b = indexesWithQueries[j]
+        const plansMatch = JSON.stringify(a.query.indexPlan) === JSON.stringify(b.query.indexPlan)
+        if(plansMatch) {
+          if(a.info.name !== b.info.name) {
+            console.log("INDEXES HAVE THE SAME PLAN", a.info.name, b.info.name, "but different names => merge")
+            b.info.name = a.info.name
+          } else {
+            console.error("INDEXES HAVE THE SAME PLAN AND NAME", a.info.name)
+          }
+          indexesWithQueries.splice(j, 1)
+          j--
+          continue
+        }
+        if(a.info.name === b.info.name) { // check for collision
+          console.error("INDEXES HAVE THE SAME NAME", a.info.name, "but different plans => error")
+          console.log("PLAN A", JSON.stringify(a.query.indexPlan, null, 2))
+          console.log("PLAN B", JSON.stringify(b.query.indexPlan, null, 2))
+          throw new Error("INDEXES HAVE THE SAME NAME " + a.info.name + " but different plans")
+        }        
+      }
+    }
+
+    for(const index of indexesWithQueries) {
+      if(this.service.indexes[index.info.name.slice(this.service.name.length+1)]) {
+        if(JSON.stringify(this.indexPlan)
+           === JSON.stringify(this.service.indexes[index.info.name.slice(this.service.name.length+1)].parameters.plan)) 
+          continue
+        console.error("INDEX", index.info.name, "ALREADY EXISTS BUT DIFFERENT PLAN")
+        throw new Error("INDEX" + index.info.name + "ALREADY EXISTS BUT DIFFERENT PLAN")
+      }
+      this.service.index({
+        name: index.info.name.slice(this.service.name.length+1),
+        function: simpleIndexCode,
+        sourceName: simpleIndexCodePath,
+        parameters: {
+          plan: index.query.indexPlan
+        }   
+      })
+    }
+  }
+
+  prepareQuery() {
+    this.createIndexes()
+
+    console.log("########### PREPARING QUERY!!!!")
+
+    this.printRules()
+
+    this.printDependencies()
 
     this.computeExecutionPlan()
     console.log("EXECUTION PLAN:")
     console.log(JSON.stringify(this.executionPlan, null, 2))
+
+    process.exit(0)
     /// TODO: create indexes used by query
 
     
     /// TODO: prepare query
 
-    process.exit(0)
+    // process.exit(0)
   }
 
   createIndex(name, mapping: OutputMapping[]) {    
@@ -547,10 +626,26 @@ export class QueryDefinition<SDS extends ServiceDefinitionSpecification> {
     this.printRules()
     console.log("OUTPUT MAPPINGS", mapping)
     this.computeIndexPlan(mapping)
-    console.log("INDEX PLAN", JSON.stringify(this.indexPlan, null, 2))
+    console.log("INDEX", name, "PLAN", JSON.stringify(this.indexPlan, null, 2))
     /// TODO: create index from query
 
-    process.exit(0)
+    if(this.service.indexes[name]) {
+      if(JSON.stringify(this.indexPlan) === JSON.stringify(this.service.indexes[name].parameters.plan)) 
+        return
+      console.error("INDEX", name, "ALREADY EXISTS BUT DIFFERENT PLAN")
+      throw new Error("INDEX" + name + "ALREADY EXISTS BUT DIFFERENT PLAN")
+    }
+
+    this.service.index({
+      name,
+      function: simpleIndexCode,
+      sourceName: simpleIndexCodePath,
+      parameters: {
+        plan: this.indexPlan
+      }   
+    })
+
+    //process.exit(0)
   }
 }
 
@@ -570,7 +665,7 @@ export default function queryFactory<SDS extends ServiceDefinitionSpecification>
   return queryFactoryFunction
 }
 
-type RuleInput = QueryInputBase | QueryPropertyBase | any
+type RuleInput = QueryInputLike | QueryPropertyBase | any
 
 function getSource(input: RuleInput): QuerySource {
   if(input instanceof QueryInputBase) return input.$source
@@ -585,12 +680,13 @@ function isStatic(element: any) {
 
 function queryDescription(element: any, indent: string = "") {  
   const flags = isStatic(element) ? "static " : ""
-  if(typeof element.toQueryDescription === "function") 
-    return flags + element.toQueryDescription(indent)
+  if(typeof element !== "object" || element === null) return flags + element.toString()
+  if(typeof element.$_toQueryDescription === "function") 
+    return flags + element.$_toQueryDescription(indent)
   const fields = Object.entries(element)
     .map(([key, value]) => `${indent}  ${key}: ${queryDescription(value, indent + "  ")}`)
     .join("\n")
-  if(element.constructor.name !== "Object") return flags + `${element.constructor.name}(${fields})`
+  if(element.constructor.name !== "Object") return flags + `${element.constructor.name}(\n${fields})`
   return flags + '{\n'+fields+`\n${indent}}`;
 }
 
@@ -609,7 +705,7 @@ function markStatic(element: any) {
 
 function parameterJSON(element: any) {
   if(typeof element !== "object" || element === null) return element
-  if(element instanceof QueryPropertyBase) return { property: element.$path }
+  if(element instanceof QueryPropertyBase) return { type: 'property', path: element.$path }
   const output = {
     type: 'object',
     properties: {}
@@ -623,6 +719,8 @@ function parameterJSON(element: any) {
 function parametersJSONForInput(input: RuleInput, resultParameters: string[]) {
   if(isStatic(input)) {
     return parameterJSON(input)        
+  } else if(typeof input.$_parametersJSON === "function") {
+    return input.$_parametersJSON(resultParameters)
   } else {
     const resultParameter = resultParameters.find(p => p === input.$alias)
     if(resultParameter) {
@@ -631,6 +729,7 @@ function parametersJSONForInput(input: RuleInput, resultParameters: string[]) {
         path: [resultParameter, ...input.$path],
       }
     }
+    //throw new Error("Result parameter not found for input: "+input.$alias)
   }
 }
 
@@ -638,13 +737,13 @@ export class RangeRule extends QueryRule {
   $input: RuleInput
   $range: RuleInput
 
-  constructor(input: QueryInputBase, range: Range) {
+  constructor(input: QueryInputLike, range: Range) {
     super()
     this.$input = input
     this.$range = range
   }
 
-  toQueryDescription(indent: string = "") {
+  $_toQueryDescription(indent: string = "") {
     return `Range(`+
            `\n${indent}  ${queryDescription(this.$input, indent + "  ")}`+
            `\n${indent}  ${queryDescription(this.$range, indent + "  ")}`+
@@ -690,7 +789,7 @@ export class EqualsRule extends QueryRule {
     this.$inputB = inputB
   }
 
-  toQueryDescription(indent: string = "") {
+  $_toQueryDescription(indent: string = "") {
     return `Equals(`+
            `\n${indent}  ${queryDescription(this.$inputA, indent + "  ")}`+
            `\n${indent}  ${queryDescription(this.$inputB, indent + "  ")}`+
@@ -731,7 +830,114 @@ function sourceType(source: QuerySource) {
     ? "table" : "index"
 }
 
-export class QueryInputBase extends CanBeStatic {
+export interface QueryInputLike {  
+  $source: QuerySource
+  $alias: string
+
+  $get(...path: string[]): QueryInputLike
+  $inside(range: Range): QueryRule
+  $equals(value: any): QueryRule
+  $concat(...input: QueryInputLike[]): QueryInputLike
+
+  $_markStatic()
+  $_isStatic()
+  $_canBeUsedAsSource(input: QueryInputLike)
+  $_getIndexInfo(indexes: IndexInfo[], serviceDefinition: ServiceDefinition<any>): IndexInfo | null
+  $_executionJSON()
+  $_equals(other: QueryInputLike)
+
+  $_toQueryDescription(indent: string): string
+}
+
+export class CompoundQueryInputBase implements QueryInputLike {
+  $source: QuerySource
+  $paths: string[][]
+  $alias: string
+
+  constructor(source: QuerySource, paths: string[][], alias: string) {    
+    this.$source = source
+    this.$paths = paths
+    this.$alias = alias
+  }
+
+  $get(...path: string[]): QueryInputLike {
+    throw new Error('compound inputs does not have fields, while fetching field: '+path.join(".")+' of '+this.$alias)
+  }
+
+  $inside(range: Range) {
+    return new RangeRule(this, range)
+  }
+
+  $equals(value: any) {
+    return new EqualsRule(this, value)
+  }
+
+  $concat(...input: QueryInputLike[]) {
+    if(!input.every(i => i.$source === this.$source)) throw new Error('concat only supports inputs from the same source')
+    return new CompoundQueryInputBase(this.$source, [...this.$paths, ...input.map(i => {
+      if(i instanceof CompoundQueryInputBase) return i.$paths
+      if(i instanceof QueryInputBase) return [i.$path]
+      throw new Error('concat only supports QueryInputBase and CompoundQueryInputBase')
+    }).flat()], this.$alias)
+  }
+
+
+  $_markStatic() {
+    /// ignore - QueryInput is not static
+  }
+
+  $_isStatic() {
+    return false
+  }
+
+  $_canBeUsedAsSource(input: QueryInputLike) {
+    return this.$source === input.$source && this.$alias === input.$alias
+  }
+
+
+  $_getIndexInfo(indexes: IndexInfo[], serviceDefinition: ServiceDefinition<any>): IndexInfo | null {
+    return new IndexInfo([
+      new RangeRule(this, {}),
+    ], [
+      ...this.$paths.map(path => new OutputMapping(this.$alias, path, path.join("_"))),      
+      new OutputMapping(this.$alias, ['id'], 'to')
+    ], serviceDefinition)
+  }
+
+  $_executionJSON() {    
+    return {
+      sourceType: sourceType(this.$source),
+      name: this.$source.getTypeName(),
+      alias: this.$alias
+    }
+  }
+  $_equals(other: QueryInputLike) {
+    if(!(other instanceof CompoundQueryInputBase)) return false
+    return this.$source === other.$source 
+      && this.$paths.length === other.$paths.length
+      && this.$paths.every((path, i) => path.join(".") === other.$paths[i].join(".")) 
+      && this.$alias === other.$alias
+  }
+
+  $_toQueryDescription(indent: string = "") {
+    return `CompoundQueryInput(\n${indent}  source: ${queryDescription(this.$source, indent + "  ")}`+
+                      `\n${indent}  paths: ${this.$paths.map(path => path.join(".")).join(", ")}`+
+                      `\n${indent}  alias: ${this.$alias}`+
+                      `\n${indent})`
+  }
+
+  $_parametersJSON(resultParameters: string[]) {
+    const resultParameter = resultParameters.find(p => p === this.$alias)    
+    if(resultParameter) {
+      return this.$paths.map(path => ({
+        type: 'result',
+        path: [resultParameter, ...path],
+      }))      
+    }
+  }
+}
+
+export class QueryInputBase extends CanBeStatic implements QueryInputLike {
   $source: QuerySource
   $path: string[]
   $alias: string
@@ -744,6 +950,38 @@ export class QueryInputBase extends CanBeStatic {
     return new EqualsRule(this, value)
   }
 
+  $get(...path: string[]): QueryInputLike {
+    return new QueryInputBase(this.$source, [...this.$path, ...path], this.$alias)
+  }
+
+  $concat(...input: QueryInputLike[]) {
+    if(!input.every(i => i.$source === this.$source)) throw new Error('concat only supports inputs from the same source')
+    return new CompoundQueryInputBase(this.$source, [this.$path, ...input.map(i => {
+      if(i instanceof CompoundQueryInputBase) return i.$paths
+      if(i instanceof QueryInputBase) return [i.$path]
+      throw new Error('concat only supports QueryInputBase and CompoundQueryInputBase')
+    }).flat()], this.$alias)
+  }
+
+  /** 
+   * @returns corresponding type property 
+   * */
+  $type() {
+    const typePath:string[] = [
+      ...this.$path.slice(0, -1), 
+      this.$path[this.$path.length - 1] + 'Type'
+    ]
+    return createQueryInputProxy(new QueryInputBase(this.$source, typePath, this.$alias))
+  }
+
+  /**
+   * Transforms property to typed property adding propertyType property
+   * @returns typed property multiple fields key
+   */
+  $typed() {
+    return this.$type().$concat(this)
+  }
+
   $as(alias: string) {
     return createQueryInputProxy(new QueryInputBase(this.$source, this.$path, alias))
   }
@@ -755,7 +993,7 @@ export class QueryInputBase extends CanBeStatic {
     this.$alias = alias
   }
 
-  toQueryDescription(indent: string = "") {
+  $_toQueryDescription(indent: string = "") {
     return `QueryInput(\n${indent}  source: ${queryDescription(this.$source, indent + "  ")}`+
                       `\n${indent}  path: ${this.$path.join(".")}`+
                       `\n${indent}  alias: ${this.$alias}`+
@@ -780,7 +1018,7 @@ export class QueryInputBase extends CanBeStatic {
     return new IndexInfo([
       new RangeRule(this, {}),
     ], [
-      new OutputMapping(this.$alias, this.$path, this.$path[this.$path.length - 1]),
+      new OutputMapping(this.$alias, this.$path, this.$path.join("_")),
       new OutputMapping(this.$alias, ['id'], 'to')
     ], serviceDefinition)
   }
@@ -789,13 +1027,23 @@ export class QueryInputBase extends CanBeStatic {
     return {
       sourceType: sourceType(this.$source),
       name: this.$source.getTypeName(),
-      path: [...this.$path],
       alias: this.$alias
     }
   }
-  $_equals(other: QueryInputBase) {
+  $_equals(other: QueryInputLike) {
+    if(!(other instanceof QueryInputBase)) return false
     return this.$source === other.$source && this.$path.join(".") === other.$path.join(".") && this.$alias === other.$alias
   }
+
+  $_parametersJSON(resultParameters: string[]) {
+    const resultParameter = resultParameters.find(p => p === this.$alias)    
+    if(resultParameter) {
+      return {
+        type: 'result',
+        path: [resultParameter, ...this.$path],
+      }
+    }
+  }
 }
 
 export class QueryInput extends QueryInputBase {
@@ -804,13 +1052,13 @@ export class QueryInput extends QueryInputBase {
 
 
 export function createQueryInputProxy(
-  base: QueryInputBase
+  base: QueryInputLike
 ) {
   return new Proxy(base, {
     get(target, prop, receiver) {
       const foundInBase = Reflect.get(target, prop, receiver)
       if(foundInBase) return foundInBase      
-      const newBase = new QueryInputBase(base.$source, [...base.$path, prop as string], base.$alias)
+      const newBase = target.$get(prop as string)
       const inputProxy = createQueryInputProxy(newBase)
       return inputProxy
     }
@@ -825,7 +1073,7 @@ export class QueryPropertyBase extends CanBeStatic {
     this.$path = path
   }
 
-  toQueryDescription(indent: string = "") {
+  $_toQueryDescription(indent: string = "") {
     return `QueryProperty(${this.$path.join(".")})`
   }
 

package/src/simpleIndex.db.js

@@ -0,0 +1,183 @@
+async function autoIndex(input, output, { plan, properties }) {
+
+  async function getFromSource(source, by) {
+    if(typeof by === 'string') return [await source.object(by).get()]
+    return await source.range(by).get()
+  }
+
+  async function fetch(planStep, context, oldContext) {
+    const { execution, next } = planStep
+    const source = await getSource(execution.sourceType, execution.name)
+    const by = context && decodeParameter(execution.by, context, properties) 
+    const oldBy = oldContext && decodeParameter(execution.by, oldContext, properties)
+    const data = await getFromSource(source, by)
+    const oldData = await getFromSource(source, oldBy)
+    context[execution.alias] = data
+    oldContext[execution.alias] = oldData    
+  }
+
+  async function gatherOutputData(next, context, oldContext) {
+    const outputContext = { ...context }
+    const oldOutputContext = { ...context }
+    /// first execute next to gather all data
+    for(const nextStep of next) {
+      await fetch(nextStep, outputContext, oldOutputContext)            
+    }
+    return [outputContext, oldOutputContext]
+  }
+
+/*   function extractMappedDataWithId(mapping, context) {
+    const outputData = {}
+    const idParts = []
+    for(const key in execution.mapping) {
+      const outputPath = execution.mapping[key]
+      outputData[key] = outputPath.reduce((acc, path) => acc?.[path], outputContext)
+      if(!outputData[key]) return null
+      idParts.push(outputData[key])
+    }
+    const id = serializeKey(idParts)
+    return { id, outputData }
+  } */
+
+  function objectPath(object, path) {
+    return path.reduce((acc, path) => acc?.[path], object)
+  }
+
+  function decodeParameter(parameter, context, properties) {
+    if(typeof parameter !== 'object') return parameter
+    if(Array.isArray(parameter)) return parameter.map(p => decodeParameter(p, context, properties))
+    if(parameter.type === 'object') return Object.fromEntries(
+      Object.entries(parameter.properties).map(([key, value]) => [key, decodeParameter(value, context, properties)])
+    )
+    if(parameter.type === 'property') return objectPath(properties, parameter.path)
+    if(parameter.type === 'result') return objectPath(context, parameter.path)
+    throw new Error(`Invalid parameter type: ${parameter.type}`)
+  }
+
+  async function getSource(sourceType, name) {
+    switch(sourceType) {
+      case 'table':
+        return await input.table(name)
+      case 'index':
+        return await input.index(name)
+      case 'log':
+        return await input.log(name)
+      default:
+        throw new Error(`Invalid source type: ${sourceType}`)
+    }  
+  }
+
+  function allInPath(path, context) {
+    if(!context) return []
+    const [ first, ...rest ] = path    
+    const values = Array.isArray(context[first]) ? context[first] : [context[first]]
+    if(rest.length === 0) return values
+    return values.map(value => allInPath(rest, value)).flat()
+  }
+
+  function generateOutputData(entries, context) {
+    const [ first, ...rest ] = entries
+    const values = allInPath(first[1], context)
+    //console.log("GENERATE OUTPUT FROM ENGRY", first, "CONTEXT", context, "VALUES", values)
+    return values.map(value => {
+      if(rest.length === 0) return [{
+        [first[0]]: value
+      }]
+      const generated = generateOutputData(rest, context)
+      return generated.map(data => ({
+        [first[0]]: value,
+        ...data
+      }))
+    }).flat()
+  }
+
+  function keyEntry(entries, entryData) {
+    const idParts = new Array(entries.length)
+    for(let i = 0; i < entries.length; i++) {      
+      idParts[i] = entryData[entries[i][0]]
+      if(!idParts[i]) return null
+    }
+    return { ...entryData, id: serializeKey(idParts) }
+  }
+
+  function keyData(entries, data) {
+    return data.map(entry => keyEntry(entries, entry)).filter(entry => entry !== null)
+  }
+
+  function findChanges(data, oldData) {
+    //output.debug("FIND CHANGES", data, "OLD DATA", oldData)
+    const byKey = new Map()
+    for(const entry of data) {    
+      if(byKey.has(entry.id)) {
+        throw new Error(`Duplicate id: ${entry.id}`)
+      } else {
+        byKey.set(entry.id, { id: entry.id, entry, oldEntry: null })
+      }      
+    }
+    for(const entry of oldData) {
+      if(byKey.has(entry.id)) {
+        byKey.get(entry.id).oldEntry = entry
+      } else {
+        byKey.set(entry.id, { id: entry.id, entry: null, oldEntry: entry })
+      }
+    }
+    return Array.from(byKey.values())
+  }
+
+  const observations = new Map()
+
+  async function execute(planStep, context, oldContext) {
+    //output.debug("EXECUTE STEP", planStep, "WITH CONTEXT", context, "AND OLD CONTEXT", oldContext)
+    const { execution, next } = planStep    
+    if(execution.operation === 'output') {
+      const [outputData, oldOutputData] = await gatherOutputData(next, context, oldContext)       
+
+      //output.debug("OUTPUT DATA", outputData, "OLD OUTPUT DATA", oldOutputData)
+
+      const mappingEntries = Object.entries(execution.mapping)
+      const mappedData = generateOutputData(mappingEntries, outputData)
+      const oldMappedData = generateOutputData(mappingEntries, oldOutputData)
+
+      //output.debug("MAPPED DATA2", mappedData, 'FROM', outputData, "MAP", mappingEntries)
+      //output.debug("OLD MAPPED DATA2", oldMappedData, 'FROM', oldOutputData, "MAP", mappingEntries)
+
+      const keyedData = keyData(mappingEntries, mappedData)
+      const oldKeyedData = keyData(mappingEntries, oldMappedData)
+
+      //output.debug("KEYED DATA", keyedData, "OLD KEYED DATA", oldKeyedData)
+
+      const changes = findChanges(keyedData, oldKeyedData)
+      for(const change of changes) {
+        await output.change(change.entry, change.oldEntry)
+      }      
+      return;
+    }
+    const source = await getSource(execution.sourceType, execution.name)
+
+    const by = context && decodeParameter(execution.by, context, properties) 
+    const oldBy = oldContext && decodeParameter(execution.by, oldContext, properties)
+    const byKey = by && serializeKey(by)
+    const oldByKey = oldBy && serializeKey(oldBy)
+    
+    if(byKey !== oldByKey) {
+      if(observations.has(oldByKey)) {
+        const observation = observations.get(oldByKey)
+        observation.dispose()
+        observations.delete(oldByKey)
+      }
+      if(!observations.has(byKey)) {
+        const observation = await source.range(by).onChange(async (obj, oldObj) => {
+          const nextContext = { ...context, [execution.alias]: obj }
+          const nextOldContext = { ...context, [execution.alias]: oldObj } 
+          // not ...oldContext, oldContext was used in previous observation
+          for(const nextStep of next) {
+            await execute(nextStep, nextContext, nextOldContext)
+          }
+        })
+        observations.set(byKey, observation)
+      }
+    }
+  }
+
+  await Promise.all(plan.map(step => execute(step, {}, null)))  
+}

package/src/simpleQuery.db.js

@@ -0,0 +1,175 @@
+async function simpleQuery(input, output, { _query, ...params }) {
+
+  const plan = _query.plan
+  const idFunction = idFunction && eval(`(${_query.idFunction})`)
+
+  function sourceChangeStream(source, by) {
+    if(typeof by === 'string') return source.object(by)
+    if(Array.isArray(by)) return source.object(serializeKey(by))
+    return source.range(by)
+  }
+
+  function objectPath(object, path) {
+    return path.reduce((acc, path) => acc?.[path], object)
+  }
+
+  function decodeParameter(parameter, context, properties) {
+    if(typeof parameter !== 'object') return parameter
+    if(Array.isArray(parameter)) return parameter.map(p => decodeParameter(p, context, properties))
+    if(parameter.type === 'object') return Object.fromEntries(
+      Object.entries(parameter.properties).map(([key, value]) => [key, decodeParameter(value, context, properties)])
+    )
+    if(parameter.type === 'property') return objectPath(properties, parameter.path)
+    if(parameter.type === 'result') return objectPath(context, parameter.path)
+    throw new Error(`Invalid parameter type: ${parameter.type}`)
+  }
+
+  async function getSource(sourceType, name) {
+    switch(sourceType) {
+      case 'table':
+        return await input.table(name)
+      case 'index':
+        return await input.index(name)
+      case 'log':
+        return await input.log(name)
+      default:
+        throw new Error(`Invalid source type: ${sourceType}`)
+    }  
+  }
+
+  class DataObservation {
+
+    #planStep = null
+    #context = null
+    #source = null
+    #by = null
+    #onChange = null
+
+    #observation = null
+
+    #dependentObservations = new Map()
+
+    #resultsPromise = null
+    #results = null
+
+    constructor(planStep, context, source, by, onChange) {
+      this.#planStep = planStep
+      this.#context = context      
+      this.#source = source
+      this.#by = by
+      this.#onChange = onChange
+    }
+
+    async start() {
+      const planStep = this.#planStep
+      const context = this.#context
+      if(!this.#source) this.#source = await getSource(planStep.execution.sourceType, planStep.execution.name)
+      if(!this.#by) this.#by = decodeParameter(planStep.execution.by, context, params)
+      let results = []      
+      const observationPromise = sourceChangeStream(this.#source, this.#by).onChange(async (obj, oldObj) => {   
+        const id = obj?.id || oldObj?.id
+        if(!id) return
+
+        if(!this.#results) { // still fetching
+          results.push(obj)
+        } else { // already fetched - maintain memory list of results
+          if(oldObj) {
+            const index = this.#results.findIndex(result => result.id === oldObj.id)
+            if(index !== -1) {
+              if(obj) {
+                this.#results[index] = obj
+              } else {
+                this.#results.splice(index, 1)
+              }              
+            }
+          } else if(obj) {
+            const index = this.#results.findIndex(result => result.id >= oldObj.id) // idempotency check
+            if(index === -1) {
+              results.push(obj)
+            } else if(this.#results[index].id === oldObj.id) {
+              this.#results[index] = obj
+            } else {
+              this.#results.splice(index, 0, obj)
+            }
+          }
+        }
+
+        const nextContext = { ...context, [planStep.execution.alias]: obj }
+        const nextOldContext = { ...context, [planStep.execution.alias]: oldObj }
+        
+        const objectObservations = []
+        
+        for(const nextStep of planStep.next) {
+          const nextSource = await getSource(nextStep.execution.sourceType, nextStep.execution.name)
+          const nextBy = decodeParameter(nextStep.execution.by, nextContext, params)
+          const nextOldBy = decodeParameter(nextStep.execution.by, nextOldContext, params)
+          const nextByKey = nextBy && serializeKey([nextStep.execution.alias, nextBy])
+          const nextOldByKey = nextOldBy && serializeKey([nextStep.execution.alias, nextOldBy])
+          if(nextByKey !== nextOldByKey) {
+            if(this.#dependentObservations.has(nextOldByKey)) {
+              const dependentObservation = this.#dependentObservations.get(nextOldByKey)
+              dependentObservation.dispose()
+              this.#dependentObservations.delete(nextOldByKey)
+            }
+            if(!this.#dependentObservations.has(nextByKey)) {
+              const dependentObservation = new DataObservation(nextStep, nextContext, nextSource, nextBy, 
+                (context, oldContext, observation) => this.handleDependentChange(context, oldContext, observation, id))
+              this.#dependentObservations.set(nextByKey, dependentObservation)
+              await dependentObservation.start()
+            }
+          }
+          if(nextByKey) objectObservations.push(this.#dependentObservations.get(nextByKey))                    
+        }
+
+        /// TODO: do object observations cross product, add self, and push change to parent
+      })
+      this.#resultsPromise = observationPromise.then(() => {
+        this.#results = results
+        return results
+      })
+      this.#observation = await observationPromise      
+    }
+
+    async results() {
+      if(!this.#results) return this.#results
+      return await this.#resultsPromise
+    }
+
+    async handleDependentChange(context, oldContext, observation, id) {
+      const currentResult = this.#results.find(result => result.id === id)
+      if(!currentResult) return /// or delete associated objects      
+      const currentContext = { [this.#planStep.execution.alias]: currentResult }
+      await this.#onChange({
+        ...currentContext,  
+        ...context,         
+        ...currentContext,       
+      }, {
+        ...currentContext,  
+        ...oldContext,
+        ...currentContext,
+      }, this)
+    }
+
+    dispose() {
+      this.#observation.dispose()
+      this.#dependentObservations.forEach(dependentObservation => dependentObservation.dispose())
+      this.#dependentObservations.clear()
+      this.#results = null
+      this.#resultsPromise = null
+      this.#observation = null
+      this.#source = null
+      this.#by = null
+      this.#planStep = null
+      this.#context = null
+    }
+  }  
+
+  const rootObservations = plan.map(step => new DataObservation(step, {}, null, null, async (newContext, oldContext, observation) => {
+    const idParts = idFunction ? idFunction(newContext) : Object.values(newContext).map(v => v?.id)
+    const id = serializeKey(idParts)
+    await output.change({ ...newContext, id }, { ...oldContext, id })
+  }))
+
+  await Promise.all(rootObservations.map(observation => observation.start()))
+
+}

package/src/autoIndex.db.js

@@ -1,3 +0,0 @@
-async function autoIndex(input, output, { }) {
-
-}