@live-change/simple-query 0.9.162 → 0.9.163

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@live-change/simple-query",
-  "version": "0.9.162",
+  "version": "0.9.163",
   "description": "",
   "main": "src/index.js",
   "scripts": {
@@ -22,7 +22,7 @@
   },
   "type": "module",
   "dependencies": {
-    "@live-change/framework": "^0.9.162",
+    "@live-change/framework": "^0.9.163",
     "pluralize": "^8.0.0"
   },
   "devDependencies": {
@@ -30,5 +30,5 @@
     "typedoc-plugin-markdown": "^4.6.3",
     "typedoc-plugin-rename-defaults": "^0.7.3"
   },
-  "gitHead": "59cd1485ca6d76bd87a6634a92d6e661e5803d5e"
+  "gitHead": "c410e1dacd07daed9a5c55367abd7f19d9a575cc"
 }

package/src/query.ts

@@ -2,7 +2,16 @@ import {
   PropertyDefinitionSpecification, ServiceDefinition, ServiceDefinitionSpecification  
 } from "@live-change/framework"
 
-import { ModelDefinition, ForeignModelDefinition } from "@live-change/framework"
+import { 
+  ModelDefinition, ForeignModelDefinition, ContextBase, QueryParameters as FrameworkQueryParameters,
+  QueryDefinition as FrameworkQueryDefinition, 
+  QueryDefinitionSpecification as FrameworkQueryDefinitionSpecification,
+  ViewDefinition,
+  EventDefinition,
+  TriggerDefinition,
+  ActionDefinition,
+  ViewDefinitionSpecificationDaoPath
+} from "@live-change/framework"
 
 import { PropertyDefinition } from "@live-change/framework"
 
@@ -17,7 +26,7 @@ const simpleQueryCode = readFileSync(simpleQueryCodePath, 'utf8')
 
 
 interface Range {
-  gt?: string
+  gt?: string 
   gte?: string
   lt?: string
   lte?: string,
@@ -51,7 +60,6 @@ class RuleSource {
   dependsOn: RuleSource[]
   index: IndexInfo | null
 
-
   constructor(rule: QueryRule, input: QueryInput, source: QuerySource, type: string) {
     this.rule = rule
     this.input = input
@@ -94,6 +102,14 @@ interface QueryDefinitionSpecification {
   sources?: Record<string, QuerySource>,
   code?: QueryCode,
   update?: boolean,
+  id: Function | string,
+  timeout?: number,
+  requestTimeout?: number,
+  validation?: (parameters: FrameworkQueryParameters, context: ContextBase) => Promise<any>
+  view?: Record<string, any>
+  event?: Record<string, any>
+  trigger?: Record<string, any>
+  action?: Record<string, any>
 }
 
 class OutputMapping {
@@ -181,7 +197,8 @@ class IndexInfo {
   $_createQuery(service: ServiceDefinition<any>) {
     return new QueryDefinition(service, {
       name: this.name,
-      properties: {}
+      properties: {},
+      id: (x:any) => x
     }, this.rules)
   }
 }
@@ -207,6 +224,8 @@ export class QueryDefinition<SDS extends ServiceDefinitionSpecification> {
   executionPlan: ExecutionStep[]
   indexPlan: ExecutionStep[]
 
+  preparedQuery: FrameworkQueryDefinition<FrameworkQueryDefinitionSpecification>
+
   constructor(
     serviceDefinition: ServiceDefinition<SDS>, definition: QueryDefinitionSpecification, rules: QueryRule[] = undefined
   ) {
@@ -340,7 +359,7 @@ export class QueryDefinition<SDS extends ServiceDefinitionSpecification> {
     for(const key in this.rules) {
       const rule = this.rules[key]
       console.log(`${indent}  RULE ${key}:`)
-      console.log(`${indent}    ${queryDescription(rule, indent + '    ')}`)      
+      console.log(`${indent}    ${queryDescription(rule, indent + '    ')}`)
       console.log(`${indent}    SOURCES:`)
       for(const ruleSource of this.ruleSources) {
         if(ruleSource.rule !== rule) continue
@@ -377,32 +396,33 @@ export class QueryDefinition<SDS extends ServiceDefinitionSpecification> {
     }
   }
 
-  computeSourceExecutionPlan(source: RuleSource, resultParameters: string[]) {
-    /// TODO: W poniższej linii jest bug, powinno pobierać odmienne source, a nie te bezpośrednio zależne.
-    /// Poza tym przy pobieraniu wstecz nie koniecznie potrzeba indexów,
-    ///  może trzeba wprowadzić index-forward i index-backward ?
-    /// A może trzeba wprowadzić analizę tego co mamy i co chcemy uzyskać w przyszłości ?
+  computeSourceExecutionPlan(source: RuleSource, resultParameters: string[]) {    
+    /// TODO: Może trzeba wprowadzić analizę tego co mamy i co chcemy uzyskać w przyszłości ?
 
     const next = source.dependentBy.map(dependent => {
       const otherSource = this.ruleSources.find(s => s.rule === dependent.rule && s != dependent)
-      return this.computeSourceExecutionPlan(otherSource, [...resultParameters, source.input.$alias])
+      const nextStep = this.computeSourceExecutionPlan(otherSource, [...resultParameters, source.input.$alias])
+      return nextStep
     })
 
     console.log("COMPUTING SOURCE EXECUTION PLAN FOR", source.input.$_toQueryDescription(), "WITH", resultParameters)
     console.log("RULE", queryDescription(source.rule, '  '))
 
     const ruleParameters = JSON.parse(JSON.stringify(source.rule.$_parametersJSON(resultParameters)))
+    console.log("RULE PARAMETERS", ruleParameters)
     if(source.index) {                  
       const indexExecution = {
         ...source.index.$_executionJSON(),
-        by: ruleParameters[Object.keys(ruleParameters)[0]]
+        by: parameterEnsureRange(ruleParameters[Object.keys(ruleParameters)[0]])
       }
       const indexNext = [{
-        operation: 'object',
-        ...source.input.$_executionJSON(),
-        by: {
-          type: 'result',
-          path: [indexExecution.alias, source.index.indexParts.at(-1).alias],
+        execution: {
+          operation: 'object',
+          ...source.input.$_executionJSON(),
+          by: {
+            type: 'result',
+            path: [indexExecution.alias, source.index.indexParts.at(-1).alias],
+          },
         },
         next
       }]
@@ -612,13 +632,34 @@ export class QueryDefinition<SDS extends ServiceDefinitionSpecification> {
     console.log("EXECUTION PLAN:")
     console.log(JSON.stringify(this.executionPlan, null, 2))
 
-    process.exit(0)
-    /// TODO: create indexes used by query
-
-    
-    /// TODO: prepare query
+    //process.exit(0)
 
-    // process.exit(0)
+    const idFunctionCode = (typeof this.definition.id === 'string')
+      ? this.definition.id : `(${this.definition.id})`
+  
+    console.log("ID FUNCTION", idFunctionCode)
+
+    this.preparedQuery = this.service.query({
+      name: this.definition.name,
+      properties: this.definition.properties,
+      returns: this.definition.returns,
+      code: simpleQueryCode,
+      sourceName: simpleQueryCodePath,
+      update: this.definition.update,
+      timeout: this.definition.timeout,
+      requestTimeout: this.definition.requestTimeout,
+      validation: this.definition.validation,
+      config: {
+        plan: this.executionPlan,
+        idFunction: idFunctionCode
+      },
+      view: this.definition.view,
+      trigger: this.definition.trigger,
+      action: this.definition.action,
+      event: this.definition.event
+    })
+ 
+    return this.preparedQuery
   }
 
   createIndex(name, mapping: OutputMapping[]) {    
@@ -649,6 +690,22 @@ export class QueryDefinition<SDS extends ServiceDefinitionSpecification> {
   }
 }
 
+function parameterIsRange(parameter: any) {
+  if(typeof parameter !== 'object' || parameter === null) return false
+  if(Array.isArray(parameter)) return parameterIsRange(parameter.at(-1))
+  if(parameter.type === 'object') return true  
+  return false
+}
+
+function parameterEnsureRange(parameter: any) {
+  if(parameterIsRange(parameter)) return parameter  
+  if(Array.isArray(parameter)) return [...parameter, { type: 'object', properties: {} }]
+  return {
+    type: 'array',
+    items: [parameter, { type: 'object', properties: {} }]
+  }  
+}
+
 
 export type QueryFactoryFunction<SDS extends ServiceDefinitionSpecification> = 
   (definition: QueryDefinitionSpecification) => QueryDefinition<SDS>
@@ -675,7 +732,9 @@ function getSource(input: RuleInput): QuerySource {
 function isStatic(element: any) {
   if(typeof element !== "object" || element === null) return true
   return element instanceof CanBeStatic ? element.$_isStatic() : 
-    (element.constructor.name === "Object" ? element[staticRuleSymbol] : false)  
+    ((element.constructor.name === "Object" || element.constructor.name === "Array") 
+      ? element[staticRuleSymbol] 
+      : false)  
 }
 
 function queryDescription(element: any, indent: string = "") {  
@@ -706,6 +765,7 @@ function markStatic(element: any) {
 function parameterJSON(element: any) {
   if(typeof element !== "object" || element === null) return element
   if(element instanceof QueryPropertyBase) return { type: 'property', path: element.$path }
+  if(Array.isArray(element)) return element.map(item => parameterJSON(item))
   const output = {
     type: 'object',
     properties: {}
@@ -768,7 +828,7 @@ export class RangeRule extends QueryRule {
   $_getSources(): RuleSource[] {
     return [
       new RuleSource(this, this.$input, getSource(this.$input), 'range')
-    ].filter(s => s.input != null)
+    ].filter(s => s.input?.$source != null)
   }
 
   $_parametersJSON(resultParameters: string[]) {
@@ -814,7 +874,7 @@ export class EqualsRule extends QueryRule {
     return [
       new RuleSource(this, this.$inputA, getSource(this.$inputA), 'object'), 
       new RuleSource(this, this.$inputB, getSource(this.$inputB), 'object')
-    ].filter(s => s.input != null)
+    ].filter(s => s.input?.$source != null)
   }
 
   $_parametersJSON(resultParameters: string[]) {

package/src/simpleQuery.db.js

@@ -1,11 +1,38 @@
 async function simpleQuery(input, output, { _query, ...params }) {
 
   const plan = _query.plan
-  const idFunction = idFunction && eval(`(${_query.idFunction})`)
+  const idFunction = _query.idFunction && eval(`(${_query.idFunction})`)
 
   function sourceChangeStream(source, by) {
+    /// TODO: support arrays that have object - prefixed range queries
     if(typeof by === 'string') return source.object(by)
-    if(Array.isArray(by)) return source.object(serializeKey(by))
+    if(Array.isArray(by)) {
+      if(typeof by.at(-1) === 'string') {
+        return source.object(serializeKey(by))
+      } else {
+        if(by.slice(0, -1).every(item => typeof item === 'string')) {
+          const prefix = serializeKeyData(by.slice(0, -1))
+          const range = by.at(-1)
+          const prefixedRange = {
+            gt: range.gt ? prefix + range.gt : undefined,
+            gte: range.gte ? prefix + range.gte : undefined,
+            lt: range.lt ? prefix + range.lt : undefined,
+            lte: range.lte ? prefix + range.lte : undefined,
+            reverse: range.reverse,
+            limit: range.limit,
+          }
+          if(!(prefixedRange.gt || prefixedRange.gte)) {
+            prefixedRange.gte = prefix
+          }
+          if(!(prefixedRange.lt || prefixedRange.lte)) {
+            prefixedRange.lte = prefix + "\xFF\xFF\xFF\xFF"
+          }
+          return source.range(prefixedRange)
+        } else {
+          throw new Error("Impossible to compute range from array: " + JSON.stringify(by))
+        }        
+      }
+    } 
     return source.range(by)
   }
 
@@ -50,14 +77,16 @@ async function simpleQuery(input, output, { _query, ...params }) {
     #dependentObservations = new Map()
 
     #resultsPromise = null
-    #results = null
+    #results = []
+    #idFunction = null
 
-    constructor(planStep, context, source, by, onChange) {
+    constructor(planStep, context, source, by, onChange, idFunction) {
       this.#planStep = planStep
       this.#context = context      
       this.#source = source
       this.#by = by
       this.#onChange = onChange
+      this.#idFunction = idFunction
     }
 
     async start() {
@@ -65,46 +94,27 @@ async function simpleQuery(input, output, { _query, ...params }) {
       const context = this.#context
       if(!this.#source) this.#source = await getSource(planStep.execution.sourceType, planStep.execution.name)
       if(!this.#by) this.#by = decodeParameter(planStep.execution.by, context, params)
-      let results = []      
-      const observationPromise = sourceChangeStream(this.#source, this.#by).onChange(async (obj, oldObj) => {   
+      //output.debug("STARTING OBSERVATION", planStep.execution.sourceType, planStep.execution.name, "BY", this.#by)
+      const sourceStream = sourceChangeStream(this.#source, this.#by)
+      const observationPromise = sourceStream.onChange(async (obj, oldObj) => {
+        /* output.debug(this.#planStep.execution.alias, "!", planStep.execution.sourceType, planStep.execution.name, "BY", this.#by,
+          'CHANGE', obj, oldObj) */
         const id = obj?.id || oldObj?.id
         if(!id) return
 
-        if(!this.#results) { // still fetching
-          results.push(obj)
-        } else { // already fetched - maintain memory list of results
-          if(oldObj) {
-            const index = this.#results.findIndex(result => result.id === oldObj.id)
-            if(index !== -1) {
-              if(obj) {
-                this.#results[index] = obj
-              } else {
-                this.#results.splice(index, 1)
-              }              
-            }
-          } else if(obj) {
-            const index = this.#results.findIndex(result => result.id >= oldObj.id) // idempotency check
-            if(index === -1) {
-              results.push(obj)
-            } else if(this.#results[index].id === oldObj.id) {
-              this.#results[index] = obj
-            } else {
-              this.#results.splice(index, 0, obj)
-            }
-          }
-        }
-
         const nextContext = { ...context, [planStep.execution.alias]: obj }
         const nextOldContext = { ...context, [planStep.execution.alias]: oldObj }
         
-        const objectObservations = []
+        //const objectObservations = []
+
+        const oldJoinedResults = oldObj ? await this.#joinResults([ oldObj ]) : []
         
         for(const nextStep of planStep.next) {
           const nextSource = await getSource(nextStep.execution.sourceType, nextStep.execution.name)
           const nextBy = decodeParameter(nextStep.execution.by, nextContext, params)
           const nextOldBy = decodeParameter(nextStep.execution.by, nextOldContext, params)
-          const nextByKey = nextBy && serializeKey([nextStep.execution.alias, nextBy])
-          const nextOldByKey = nextOldBy && serializeKey([nextStep.execution.alias, nextOldBy])
+          const nextByKey = nextBy && serializeKeyData([nextStep.execution.alias, nextBy])
+          const nextOldByKey = nextOldBy && serializeKeyData([nextStep.execution.alias, nextOldBy])
           if(nextByKey !== nextOldByKey) {
             if(this.#dependentObservations.has(nextOldByKey)) {
               const dependentObservation = this.#dependentObservations.get(nextOldByKey)
@@ -118,36 +128,110 @@ async function simpleQuery(input, output, { _query, ...params }) {
               await dependentObservation.start()
             }
           }
-          if(nextByKey) objectObservations.push(this.#dependentObservations.get(nextByKey))                    
+          //if(nextByKey) objectObservations.push(this.#dependentObservations.get(nextByKey))                    
+        }        
+
+        const newJoinedResults = obj ? await this.#joinResults([ obj ]) : []
+
+        //output.debug(this.#planStep.execution.alias, "oldJoinedResults", oldJoinedResults, "from", oldObj)
+        //output.debug(this.#planStep.execution.alias, "newJoinedResults", newJoinedResults, "from", obj)        
+
+        for(const oldJoinedResult of oldJoinedResults) {
+          if(!oldJoinedResult) throw new Error("oldJoinedResult is null")
+          if(!newJoinedResults.some(newResult => newResult.id === oldJoinedResult.id)) 
+            this.#joinedChange(null, oldJoinedResult)
+        }
+        for(const newJoinedResult of newJoinedResults) {
+          const oldJoinedResult = oldJoinedResults.find(oldResult => oldResult.id === newJoinedResult.id)
+          if(oldJoinedResult) {
+            if(JSON.stringify(newJoinedResult) !== JSON.stringify(oldJoinedResult)) {
+              this.#joinedChange(newJoinedResult, oldJoinedResult)
+            }
+          } else {
+            this.#joinedChange(newJoinedResult, null)
+          }
         }
 
-        /// TODO: do object observations cross product, add self, and push change to parent
       })
       this.#resultsPromise = observationPromise.then(() => {
-        this.#results = results
-        return results
+        return this.#results
       })
       this.#observation = await observationPromise      
     }
 
+    async #joinedChange(obj, oldObj, observation) {
+      if(oldObj) {
+        const index = this.#results.findIndex(result => result.id === oldObj.id)
+        if(index !== -1) {
+          if(obj) {
+            this.#results[index] = obj
+          } else {
+            this.#results.splice(index, 1)
+          }              
+        }
+      } else if(obj) {
+        const index = this.#results.findIndex(result => result.id >= obj.id)
+        if(index === -1) {
+          this.#results.push(obj)
+        } else if(this.#results[index].id === obj.id) {
+          this.#results[index] = obj
+        } else {
+          this.#results.splice(index, 0, obj)
+        }
+      }
+      await this.#onChange(obj, oldObj, this)
+    }
+
+    async #joinResults(results) {
+      let joinedResults = results.map(result => ({
+        __result_id_patrs: [result.id],
+        [this.#planStep.execution.alias]: result
+      }))
+      // output.debug("joined results", joinedResults)
+      for(const dependentObservation of this.#dependentObservations.values()) {
+        const dependentResults = await dependentObservation.results()
+        // output.debug("  dep results", dependentObservation.#planStep.execution.alias, dependentResults)
+        joinedResults = joinedResults.flatMap(
+          (joinedResult) => {            
+            if(dependentResults.length === 0) return [
+              { /// TODO: check if optional (not mandatory)
+                ...joinedResult,
+                [dependentObservation.#planStep.execution.alias]: null
+              }
+            ]
+            return dependentResults.map(dependentJoinedResult => ({            
+              ...dependentJoinedResult,
+              ...joinedResult,
+              __result_id_patrs: joinedResult.__result_id_patrs.concat(dependentJoinedResult.__result_id_patrs)
+            }))
+          }                     
+        )                 
+      }      
+      //output.debug("joinedResultsAfterDependencies", joinedResults)
+      for(const joinedResult of joinedResults) {        
+        joinedResult.id = serializeKey(
+          this.#idFunction ? this.#idFunction(joinedResult) : joinedResult.__result_id_patrs
+        )
+      }
+      return joinedResults
+    }
+
     async results() {
       if(!this.#results) return this.#results
       return await this.#resultsPromise
     }
 
     async handleDependentChange(context, oldContext, observation, id) {
-      const currentResult = this.#results.find(result => result.id === id)
-      if(!currentResult) return /// or delete associated objects      
-      const currentContext = { [this.#planStep.execution.alias]: currentResult }
-      await this.#onChange({
-        ...currentContext,  
-        ...context,         
-        ...currentContext,       
-      }, {
-        ...currentContext,  
-        ...oldContext,
-        ...currentContext,
-      }, this)
+      if(!this.#results) return 
+      // we need to find thre results affected by the change
+      for(const result of this.#results) { // bunary search can be used here for better performance
+        if(result.__result_id_patrs[0] === id) {          
+          const oldResult = { ...result }
+          const alias = observation.#planStep.execution.alias
+          result[alias] = context[alias]
+          this.#onChange(result, oldResult, observation) /// TODO: handle cases when part is mandatory
+        }
+      }
     }
 
     dispose() {
@@ -164,12 +248,21 @@ async function simpleQuery(input, output, { _query, ...params }) {
     }
   }  
 
-  const rootObservations = plan.map(step => new DataObservation(step, {}, null, null, async (newContext, oldContext, observation) => {
-    const idParts = idFunction ? idFunction(newContext) : Object.values(newContext).map(v => v?.id)
-    const id = serializeKey(idParts)
-    await output.change({ ...newContext, id }, { ...oldContext, id })
-  }))
+  const rootObservations = plan.map(step => new DataObservation(step, {}, null, null, 
+    (result, oldResult, observation) => {
+      if(result) delete result.__result_id_patrs
+      if(oldResult) delete oldResult.__result_id_patrs
+      output.change(result, oldResult)
+    },
+    idFunction
+    /* async (newContext, oldContext, observation) => {
+      const idParts = idFunction ? idFunction(newContext) : Object.values(newContext).map(v => v?.id)
+      const id = serializeKey(idParts)
+      await output.change({ ...newContext, id }, { ...oldContext, id })
+    } */
+  ))
 
   await Promise.all(rootObservations.map(observation => observation.start()))
+  await Promise.all(rootObservations.map(observation => observation.results()))
 
 }