entodicton 9.7.1-beta.1 → 9.7.1-beta.11

package/client.js

@@ -12,7 +12,7 @@ const _ = require('lodash')
 const stringify = require('json-stable-stringify')
 const Lines = require('./lines')
 const flattens = require('./src/flatten')
-const { sortJson, appendNoDups, InitCalls, updateQueries, safeNoDups, stableId, where, suggestAssociationsFix, suggestAssociationsFixFromSummaries, validProps } = require('./src/helpers')
+const { helpDebugHierarchy, sortJson, appendNoDups, InitCalls, updateQueries, safeNoDups, stableId, where, suggestAssociationsFix, suggestAssociationsFixFromSummaries, validProps, OverrideCheck, } = require('./src/helpers')
 const runtime = require('./runtime')
 const db = require('./src/debug')
 
@@ -479,7 +479,7 @@ const getConfigForTest = (config, testConfig) => {
   return configForTest
 }
 
-const runTest = async (config, expected, { args, verbose, testConfig, debug, timings = { server: 0, client: 0 } }) => {
+const runTest = async (config, expected, { args, verbose, testConfig, debug, timings = { server: 0, client: 0 }, index = 0 }) => {
   const test = expected.query
   if (args.query && args.query != test) {
     // no run this
@@ -512,10 +512,11 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug, tim
       defaultInnerProcess(config, errorHandler, result)
     }
     if (verbose) {
-      const widths = [100, 60]
+      const widths = [4, 100, 60]
       const lines = new Lines(widths)
-      lines.setElement(0, 0, test)
-      lines.setElement(0, 1, `time on server: ${result.times.toFixed(2)} client: ${(result.clientSideTimes / 1000).toFixed(2)}`)
+      lines.setElement(0, 0, index)
+      lines.setElement(0, 1, test)
+      lines.setElement(0, 2, `time on server: ${result.times.toFixed(2)} client: ${(result.clientSideTimes / 1000).toFixed(2)}`)
       timings.server += result.times
       timings.client += result.clientSideTimes / 1000
       lines.log()
@@ -618,18 +619,22 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug, tim
 
 const runTestsHelper = async (config, tests, failed, juicyBits) => {
   const { stopAtFirstError } = juicyBits
+  let index = 0
   while (true) {
     if (tests.length === 0) {
       return Promise.resolve(failed)
     }
     const test = tests.shift()
+    juicyBits.index = index
     const result = await runTest(config, test, juicyBits)
     if (result != null) {
+      result.index = index
       failed.push(result)
       if (stopAtFirstError) {
         return failed
       }
     }
+    index += 1
   }
 }
 
@@ -678,13 +683,6 @@ const saveTests = (config, testFile, testConfig) => {
   return saveTestsHelper(testFile, config, tests, tests.map((test) => test.query), testConfig)
 }
 
-/*
-const showExamples = (testFile) => {
-  const tests = JSON.parse(fs.readFileSync(testFile))
-  Object.keys(tests).forEach((test) => console.log(test))
-}
-*/
-
 const showInfo = (description, section, config) => {
   console.log(JSON.stringify(config.getInfo(), null, 2))
 }
@@ -1314,7 +1312,6 @@ const knowledgeModuleImpl = async ({
 
       const helpDebugWord = 'In order to get a debug break when a specific word is created set the DEBUG_WORD environment variable to the JSON of the association to break on. For example DEBUG_WORD=\'"the"\''
       const helpDebugAssociation = 'In order to get a debug break when a specific association is created set the DEBUG_ASSOCIATION environment variable to the JSON of the association to break on. For example DEBUG_ASSOCIATION=\'[["the", 0], ["mammal", 1]]\''
-      const helpDebugHierarchy = 'In order to get a debug break when a specific hierarchy is created set the DEBUG_HIERARCHY environment variable to the JSON of the child-parent pair to break on. For example DEBUG_HIERARCHY=\'[["cat", 1], ["mammel", 1]]\''
       const helpDebugPriority = 'In order to get a debug break when a specific set of priorities is created set set DEBUG_PRIORITY environment variable to the JSON of the priorities that you want to break on. For example DEBUG_PRIORITY=\'[["verb", 0], ["article", 0]]\''
       const helpDebugContextualPriority = 'In order to get a debug break when a specific set of contextual priorities is created set set DEBUG_CONTEXTUAL_PRIORITY environment variable to the JSON of the priorities that you want to break on. For example DEBUG_CONTEXTUAL_PRIORITY=\'{ context: [["verb", 0], ["article", 0], select: 1}\''
       const helpDebugBridge = 'In order to get a debug break when a specific bridge is created set the DEBUG_BRIDGE environment variable to id to break on. For example DEBUG_BRIDGE=\'car\''
@@ -1528,7 +1525,7 @@ const knowledgeModuleImpl = async ({
             console.log('Test queries')
             let counter = 0
             for (const test of config.tests) {
-              console.log(`${counter} - ${test.query}`)
+              console.log(`#${counter} - ${test.query}`)
               counter += 1
             }
           }
@@ -1757,7 +1754,7 @@ const knowledgeModuleImpl = async ({
                 if (hasError) {
                   console.log('**************************** ERRORS ************************')
                   for (const result of results) {
-                    console.log('Utterance: ', result.utterance)
+                    console.log(`Utterance: #${result.index} ${result.utterance}`)
                     if (!result.hasError) {
                       continue
                     }
@@ -1773,7 +1770,7 @@ const knowledgeModuleImpl = async ({
                         newError = true
                         headerShown = true
                         if (args.vimdiff) {
-                          vimdiff(actual, expected, `"${label} - ${result.utterance}"`)
+                          vimdiff(actual, expected, `"${label} - #${result.index} ${result.utterance}"`)
                         }
                         result.hasError = true
                       }
@@ -1916,12 +1913,17 @@ const knowledgeModuleImpl = async ({
                     console.log('use -v arg to write files expected.json and actual.json in the current directory for detailed comparison. Or do -s and then git diff the changes.')
                     // console.log(JSON.stringify(contexts))
                     let errorCount = 0
+                    const failedTests = []
                     for (const result of results) {
                       if (result.hasError) {
-                        console.log(`FAILED ${result.utterance}`)
+                        console.log(`FAILED #${result.index} ${result.utterance}`)
+                        failedTests.push({ index: result.index, utterance: result.utterance })
                         errorCount += 1
                       }
                     }
+                    const failedFile = './FAILED.json'
+                    console.log(`Failed tests are written to ${failedFile}`)
+                    runtime.fs.writeFileSync(failedFile, JSON.stringify(failedTests, 0, 2))
                     console.log(`**************************** THERE WERE ${errorCount} TEST FAILURES ************************`)
                   }
                 }
@@ -2104,4 +2106,5 @@ module.exports = {
   writeTest,
   getConfigForTest,
   debug: db,
+  OverrideCheck,
 }

package/index.js

@@ -1,7 +1,7 @@
 const { Semantics, Semantic } = require('./src/semantics')
 const { Generators, Generator } = require('./src/generators')
 const { Config } = require('./src/config')
-const { w, where } = require('./src/helpers')
+const { w, where, OverrideCheck } = require('./src/helpers')
 const Digraph = require('./src/digraph')
 const client = require('./client')
 const flattens = require('./src/flatten')
@@ -28,5 +28,6 @@ module.exports = {
   flattens: flattens.flattens,
   flatten: flattens.flatten,
   unflatten: unflatten.unflatten,
-  debug
+  debug,
+  OverrideCheck,
 }

package/package.json

@@ -73,6 +73,6 @@
     "scriptjs": "^2.5.9",
     "uuid": "^8.3.2"
   },
-  "version": "9.7.1-beta.1",
+  "version": "9.7.1-beta.11",
   "license": "UNLICENSED"
 }

package/src/config.js

@@ -454,7 +454,7 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
           // match: async (args) => bridge.id === args.context.marker && args.context.level === level && !args.context.paraphrase && (args.context.response || args.context.isResponse) && await match(args),
           match: async (args) => {
             if (bridge.callId == args.callId) {
-              debugger
+              debugger // eslint-disable-line no-debugger
             }
             return args.isA(args.context.marker, bridge.id) && args.context.level === level && !args.context.paraphrase && (args.context.response != null || args.context.isResponse) && await match(args)
           },
@@ -572,6 +572,7 @@ const handleCalculatedProps = (baseConfig, moreConfig, { addFirst, uuid } = {})
         'operator', 
         'optional', 
         'parents',
+        'preferOver',
         'return_type_selector', 
         'scope', 
         'selector', 
@@ -601,7 +602,12 @@ const handleCalculatedProps = (baseConfig, moreConfig, { addFirst, uuid } = {})
 }
 
 if (runtime.process.env.DEBUG_HIERARCHY) {
-  global.entodictonDebugHierarchy = JSON.parse(runtime.process.env.DEBUG_HIERARCHY)
+  try {
+    global.entodictonDebugHierarchy = JSON.parse(runtime.process.env.DEBUG_HIERARCHY)
+  } catch( e ) {
+    console.error("Invalid valid for DEBUG_HIERARCHY")
+    console.error(helpers.helpDebugHierarchy)
+  }
 }
 
 // i keep randomly doing one of the other so I will just make both work the same way
@@ -1655,6 +1661,7 @@ class Config {
     }
   }
 
+  // handler/listener
   updateBridge(id, updater) {
     const bridge = this.config.bridges.find((b) => b.id === id)
     updater({ config: this, bridge })
@@ -2313,6 +2320,7 @@ class Config {
     }
   }
 
+  // TODO Api or API -> pick one and do that
   async setMultiApi (initializer) {
     await this.setApi(() => multiApiImpl(initializer))
   }
@@ -2420,11 +2428,17 @@ class Config {
         if (!api.initialize) {
           throw new Error(`Expected the API for ${this.name} to have an initialize function.`)
         }
+        if (api.baseAPI) {
+          new helpers.OverrideCheck(api.baseAPI).check(api)
+        }
       }
     } else {
       if (!value.initialize) {
         throw new Error(`Expected the API to have an initialize function for ${this.name}. If you are trying to pass in multiple API configs at once use setApiKMs to set the names of the configs that you want set.`)
       }
+      if (value.baseAPI) {
+        new helpers.OverrideCheck(value.baseAPI).check(value)
+      }
     }
 
     if (this._api && this._api.multiApi) {

package/src/configHelpers.js

@@ -145,6 +145,7 @@ const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
     args.objects = config.get('objects')
     args.getObjects = getObjects(args.objects)
   }
+  args.debug = debug
   args.enable = (key) => {
     config.addEnable(key)
   }
@@ -163,7 +164,19 @@ const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
     }
   }
   args.contextHierarchy = new ContextHierarchy()
-  args.handlerStack = new HandlerStack()
+  args.namespaced = {
+    get: (km, context, property) => {
+      context.namespaced ??= {}
+      context.namespaced[km] ??= {}
+      return context.namespaced[km][property] 
+    },
+    set: (km, context, property, value) => {
+      context.namespaced ??= {}
+      context.namespaced[km] ??= {}
+      return context.namespaced[km][property] = value
+    }
+  }
+  args.stack = new HandlerStack()
   args.cleanAssign = cleanAssign
   args.km = (name) => config.getConfig(name)
   args.api = (name) => config.getConfig(name).api

package/src/debug.js

@@ -27,6 +27,18 @@ const unhit = (name) => {
   hits[name] = false
 }
 
+const hitScoped = async (name, call) => {
+  hit(name)
+  try {
+    const result = await call()
+    unhit(name)
+    return result
+  } catch( e ) {
+    unhit(name)
+    throw e
+  }
+}
+
 const wasHit = (name) => {
   return hits[name]
 }
@@ -64,16 +76,19 @@ const _break = (name) => {
   }
 }
 
+const breakAt = _break;
+
 module.exports = {
   counter,
   get,
   _break, // break is a keyword
-
+  breakAt,
   // used for tests
   reset,
   hit,
   unhit,
   wasHit,
   getDb,
-  setDb
+  setDb,
+  hitScoped,
 }

package/src/flatten.js

@@ -1,7 +1,8 @@
-const { isArray, isObject, isCompound } = require('./helpers')
+const { isArray, isObject, isCompound, } = require('./helpers')
+const debug = require('./debug')
 const _ = require('lodash')
 
-const DEBUG = (...args) => {
+const DEBUG = (label, details) => {
 //  DEBUG(...args)
 }
 
@@ -46,33 +47,24 @@ const zip = (a1, a2) => {
 }
 
 const flattenListHelper = (markers, values) => {
-  const flats = []
   let wasFlattened = false
-  let singles = []
+  const all = []
   for (const value of values) {
     if (!isCompound(value)) {
-      singles.push(value)
+      all.push([value])
     } else {
       const [more, wf] = flatten(markers, value)
       if (wf) {
         wasFlattened = wasFlattened || wf
-        flats.push(more)
+        all.push(more)
       } else {
-        singles = singles.concat(more)
+        all.push(...more.map((m) => [m]))
       }
     }
   }
 
-  // product does weirdness with only choice
-  if (!wasFlattened) {
-    return [[singles], wasFlattened]
-  } else {
-    const results = []
-    for (const flat of cartesian(...flats)) {
-      results.push(singles.concat(flat))
-    }
-    return [results, wasFlattened]
-  }
+  const result = cartesian(...all)
+  return [result, wasFlattened]
 }
 
 // returns (list, wasFlattened)

package/src/fragments.js

@@ -1,6 +1,29 @@
 const _ = require('lodash')
 const helpers = require('./helpers')
 
+function createChangeTracker(obj) {
+  const changes = new Set();
+
+  const handler = {
+    set(target, prop, value) {
+      changes.add(prop);
+      target[prop] = value;
+      return true;
+    },
+    deleteProperty(target, prop) {
+      changes.add(prop);
+      delete target[prop];
+      return true;
+    }
+  };
+
+  return {
+    proxy: new Proxy(obj, handler),
+    getChanges: () => Array.from(changes)
+  };
+}
+
+
 function fragmentInstantiator (args, contexts) {
   return new Object({ 
     contexts: () => {
@@ -13,13 +36,15 @@ function fragmentInstantiator (args, contexts) {
       args.pathEquals = helpers.pathEquals
       while (todo.length > 0) {
         const { context, path } = todo.pop()
-        args.context = context
+        const { proxy, getChanges } = createChangeTracker(context);
+        args.context = proxy
         args.path = path
         for (const mapping of mappings) {
           if (await mapping.match(args)) {
             await mapping.apply(args)
           }
         }
+        const ignore = getChanges()
         for (const key of Object.keys(context)) {
           // if (['number', 'string', 'boolean'].includes(typeof (context[key]))) {
           if (!helpers.isCompound(context[key])) {
@@ -28,7 +53,9 @@ function fragmentInstantiator (args, contexts) {
           if (context[key].instantiated) {
             continue
           }
-          todo.push({ context: context[key], path: [...path, key] })
+          if (!ignore.includes(key)) {
+            todo.push({ context: context[key], path: [...path, key] })
+          }
         }
       }
       if (contexts.length == 1 && instantiated.length == 1) {

package/src/generators.js

@@ -1,6 +1,7 @@
 const { args: contextArgs, normalizeGenerator } = require('./helpers')
 const Lines = require('../lines')
 const helpers = require('./helpers')
+const debug = require('./debug')
 
 class Generator {
   // constructor ({ match, apply, uuid, index, km, priority, notes }) {
@@ -65,14 +66,16 @@ class Generator {
     }
     const args = Object.assign({}, baseArgs, moreArgs, (baseArgs.getUUIDScoped || (() => { return {} }))(this.uuid))
     // return this.match(args)
-    const matches = await this.match(args)
-    if ((matches && (options.debug || {}).match) ||
-        callId === this.callId) {
-      // next line is the matcher
-      debugger // eslint-disable-line no-debugger
-      await this.match(args)
-    }
-    return matches
+    return await debug.hitScoped(args.callId, async () => {
+      const matches = await this.match(args)
+      if ((matches && (options.debug || {}).match) ||
+          callId === this.callId) {
+        // next line is the matcher
+        debugger // eslint-disable-line no-debugger
+        await this.match(args)
+      }
+      return matches
+    })
   }
 
   // apply (baseArgs, objects, g, gs, context, hierarchy, config, response, log, options = {}) {
@@ -121,11 +124,13 @@ class Generator {
     if (this.property === 'generatorp') {
       args.g = args.gp
     }
-    if ((options.debug || {}).apply ||
-        callId === this.callId) {
-      debugger // eslint-disable-line no-debugger
-    }
-    return await this._apply(args)
+    return await debug.hitScoped(args.callId, async () => {
+      if ((options.debug || {}).apply ||
+          callId === this.callId) {
+        debugger // eslint-disable-line no-debugger
+      }
+      return await this._apply(args)
+    })
   }
 }
 
@@ -220,7 +225,7 @@ class Generators {
           lines.setElement(0, 2, stack)
           lines.newRow()
           lines.setElement(0, 1, 'DEBUG')
-          lines.setElement(0, 2, `To debug this use args.callId === '${args.calls.current()}'`)
+          lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}')`)
           lines.newRow()
           lines.setElement(0, 1, 'ERROR')
           lines.setElement(0, 2, errorMessage)
@@ -252,7 +257,7 @@ class Generators {
           lines.setElement(0, 2, stack)
           lines.newRow()
           lines.setElement(0, 1, 'DEBUG')
-          lines.setElement(0, 2, `To debug this use args.callId === '${args.calls.current()}'`)
+          lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}')`)
           lines.newRow()
           lines.setElement(0, 1, 'TO')
           lines.setElement(0, 2, `context_id: ${context.context_id}`)

package/src/helpers.js

@@ -1,6 +1,27 @@
 const deepEqual = require('deep-equal')
 const stringify = require('json-stable-stringify')
 
+
+// check if member function of a class have been overriden by an subclass
+class OverrideCheck {
+  constructor(base, suffix='API') {
+    this.base = base
+    this.checks = Object.getOwnPropertyNames(base.prototype).filter(key => typeof base.prototype[key] === 'function' && key.endsWith(suffix));
+  }
+
+  check(obj) {
+    const overriden = obj.constructor !== obj.baseAPI
+    if (!overriden) {
+      return
+    }
+    for (const check of this.checks) {
+      if (obj[check] == this.base.prototype[check]) {
+        throw new Error(`For ${obj.constructor.name} you need to override ${check}`)
+      }
+    }
+  }
+}
+
 function pathEquals (p1, p2) {
   if (p1.length !== p2.length) {
     return false
@@ -579,7 +600,11 @@ function setByPath(obj, path, value) {
   return value;
 }
 
+const helpDebugHierarchy = 'In order to get a debug break when a specific hierarchy is created set the DEBUG_HIERARCHY environment variable to the JSON of the child-parent pair to break on. For example DEBUG_HIERARCHY=\'["cat", "mammel"]\''
+
 module.exports = {
+  helpDebugHierarchy,
+
   stableId,
   ecatch,
   functionsToStrings,
@@ -611,4 +636,5 @@ module.exports = {
   assignAssumed,
   watchProperty,
   pathEquals,
+  OverrideCheck,
 }

package/src/semantics.js

@@ -75,13 +75,15 @@ class Semantic {
   async matches (args, context, options = {}) {
     args = {...args}
     this.fixUpArgs(args, context)
-    const matches = await this.matcher(args)
-    if (matches && (options.debug || {}).match || args.callId === this.callId) {
-      // next line is the matcher
-      debugger // eslint-disable-line no-debugger
-      await this.matcher(args)
-    }
-    return matches
+    return await debug.hitScoped(args.callId, async () => {
+      const matches = await this.matcher(args)
+      if (matches && (options.debug || {}).match || args.callId === this.callId) {
+        // next line is the matcher
+        debugger // eslint-disable-line no-debugger
+        await this.matcher(args)
+      }
+      return matches
+    })
   }
 
   async apply (args, context, s, options = {}) {
@@ -97,21 +99,24 @@ class Semantic {
     const contextPrime = Object.assign({}, context)
     this.fixUpArgs(args, contextPrime)
 
-    if ((options.debug || {}).apply || args.callId === this.callId) {
-      debugger // eslint-disable-line no-debugger
-    }
-    if (args.breakOnSemantics) {
-      debugger // eslint-disable-line no-debugger
-    }
-    try {
-      args.handlerStack.push(this)
-      await this._apply(args)
-      args.handlerStack.pop()
-    } catch( e ) {
-      args.handlerStack.pop()
-      throw e
-    }
-    return contextPrime
+    return await debug.hitScoped(args.callId, async () => {
+      if ((options.debug || {}).apply || args.callId === this.callId) {
+        debugger // eslint-disable-line no-debugger
+      }
+      if (args.breakOnSemantics) {
+        debugger // eslint-disable-line no-debugger
+      }
+
+      try {
+        args.stack.push(this)
+        await this._apply(args)
+        args.stack.pop()
+      } catch( e ) {
+        args.stack.pop()
+        throw e
+      }
+      return contextPrime
+    })
   }
 }
 
@@ -187,6 +192,8 @@ class Semantics {
     let seenQuestion = false
     const deferred = []
     args.log = (message) => { this.logs.push(message) }
+    const finallyList = []; // called after the semantic is processed
+    args._finally = (call) => finallyList.unshift(call)
     for (const isemantic in this.semantics) {
       const semantic = this.semantics[isemantic]
       // only one question at a time
@@ -253,7 +260,7 @@ class Semantics {
           lines.setElement(0, 2, semantic.toString())
           lines.newRow()
           lines.setElement(0, 1, 'DEBUG')
-          lines.setElement(0, 2, `To debug this use args.callId === '${args.calls.current()}'`)
+          lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}')`)
           lines.newRow()
           lines.setElement(0, 1, 'TO')
           lines.setElement(0, 2, `context_id: ${context.context_id}`)
@@ -288,7 +295,7 @@ class Semantics {
           lines.setElement(0, 2, semantic.toString())
           lines.newRow()
           lines.setElement(0, 1, 'DEBUG')
-          lines.setElement(0, 2, `To debug this use args.callId === '${args.calls.current()}'`)
+          lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}')`)
           lines.newRow()
           lines.setElement(0, 1, 'TO')
           lines.setElement(0, 2, `context_id: ${context.context_id}`)
@@ -319,6 +326,9 @@ class Semantics {
       }
       counter += 1
     }
+    for (const f of finallyList) {
+      await f()
+    }
     args.calls.pop()
     if (!applied && debug) {
       const widths = Lines.addRemainder([10, 10])