entodicton 9.7.1-beta.9 → 9.8.0-beta.0

package/client.js

@@ -12,7 +12,7 @@ const _ = require('lodash')
 const stringify = require('json-stable-stringify')
 const Lines = require('./lines')
 const flattens = require('./src/flatten')
-const { helpDebugHierarchy, sortJson, appendNoDups, InitCalls, updateQueries, safeNoDups, stableId, where, suggestAssociationsFix, suggestAssociationsFixFromSummaries, validProps, OverrideCheck, } = require('./src/helpers')
+const { helpDebugHierarchy, sortJson, appendNoDups, InitCalls, updateQueries, safeNoDups, stableId, where, suggestAssociationsFix, suggestAssociationsFixFromSummaries, validProps, OverrideCheck, removeProp, } = require('./src/helpers')
 const runtime = require('./runtime')
 const db = require('./src/debug')
 
@@ -59,20 +59,11 @@ const pickObjects = (config, testConfig, getObjects) => {
       throw new Error(`In the checks for ${config.name} the KM ${km} does not exist`)
     }
 
-    if (false) {
-      if (checks[km] && checks[km].find((check) => check.match && check.apply)) {
-        projection[km] = project2(objects, checks[km])
-      } else {
-        projection[km] = project(objects, checks[km])
-      }
-    } else {
-      const lastIndex = contextChecks.length - 1
-      const last = contextChecks[lastIndex]
-      contextChecks[lastIndex] = { match: last.match, apply: () => [...new Set([...(checks[km] || []), ...last.apply()])] }
+    const lastIndex = contextChecks.length - 1
+    const last = contextChecks[lastIndex]
+    contextChecks[lastIndex] = { match: last.match, apply: () => [...new Set([...(checks[km] || []), ...last.apply()])] }
 
-      // projection[km] = project2(objects, [...checks[km], ...contextChecks])
-      projection[km] = project2(objects, contextChecks)
-    }
+    projection[km] = project2(objects, contextChecks)
   }
   return projection
 }
@@ -167,10 +158,12 @@ const writeTest = (fn, query, objects, generated, paraphrases, responses, contex
   if (runtime.fs.existsSync(fn)) {
     tests = JSON.parse(runtime.fs.readFileSync(fn))
   }
+  /*
   for (const association of associations) {
     association.sort()
   }
   associations.sort()
+  */
   // tests[query] = sortJson({ paraphrases, responses, contexts, objects: convertToStable(objects), associations, metadata, config, developerTest: saveDeveloper }, { depth: 25 })
   const results = sortJson({
     query,
@@ -265,7 +258,7 @@ const throwErrorHandler = (error) => {
   throw error
 }
 
-const _process = async (config, query, { initializer, commandLineArgs, credentials, writeTests, isProcess, isModule, isTest, saveDeveloper, rebuildingTemplate, testConfig, testsFN, errorHandler = throwErrorHandler } = {}) => {
+const _process = async (config, query, { queryNumber, initializer, commandLineArgs, credentials, writeTests, logs, isProcess, isModule, isTest, saveDeveloper, rebuildingTemplate, testConfig, testsFN, errorHandler = throwErrorHandler } = {}) => {
   if (credentials) {
     config.server(credentials.server, credentials.key)
   }
@@ -296,11 +289,12 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
   // '\\n' from tests of '\n' from users. the former is because newline is not a valid json character so the testfile has it encoded
   let queries = query.split(/\\n|\n/)
   const summaries = [] // for error
+  logs = logs || []
   try {
     const response = {
       hierarchy: [],
       load_cache_time: 0.0,
-      logs: [],
+      logs: logs,
       metadata: {
         opChoices: []
       },
@@ -334,6 +328,9 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
       data.utterance = queries[0]
       data.start_counter = startCounter
       let json = await doWithRetries(retries, url, queryParams, data)
+      if (json.logs) {
+        logs.push(...json.logs)
+      }
       let resetData = false
       if (json.code === 'NOT_IN_CACHE') {
         resetData = true
@@ -367,28 +364,27 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
         const summary = { summaries: json.summaries, length: json.contexts.length }
         summaries.push(summary)
         const { updatedContextIdCounter, contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime } =
-                                  await processContextsB({ contextIdCounter, calls, isTest, isProcess, isModule, rebuildingTemplate, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
+                                  await processContextsB({ contextIdCounter, logs, calls, isTest, isProcess, isModule, rebuildingTemplate, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
         contextIdCounter = updatedContextIdCounter
         if (isTest) {
           const end = runtime.performance.performance.now()
           clientSideTime = end - start
         }
-        response.associations = json.associations
+        // response.associations = json.associations
         response.learned_contextual_priorities = json.learned_contextual_priorities
         response.hierarchy = json.hierarchy
         response.load_cache_time += json.load_cache_time
-        appendNoDups(response.logs, json.logs)
         response.memory_free_percent = json.memory_free_percent
-        // appendNoDups(response.metadata.associations, json.metadata.associations)
-        // appendNoDups(response.metadata.priorities, json.metadata.priorities)
         appendNoDups(response.metadata.opChoices, json.metadata.opChoices)
         response.times += json.times
         response.clientSideTimes += clientSideTime
         response.trace = response.trace.concat(json.trace)
         response.version = json.version
         response.explain_priorities = json.explain_priorities
+        response.explain_associations_details = (response.explain_associations_details || []).concat(json.explain_associations_details)
+        response.explain_associations_trie = json.explain_associations_trie
         response.contextual_priorities_ambiguities = json.contextual_priorities_ambiguities
-        response.rtf_associations = json.rtf_associations
+        // response.rtf_associations = json.rtf_associations
 
         response.contexts = response.contexts.concat(contextsPrime)
         response.generated = response.generated.concat(generatedPrime)
@@ -414,6 +410,7 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
     return response
   } catch (error) {
     error.summaries = summaries
+    error.queryNumber = queryNumber
     error.query = query
     errorHandler(error)
   }
@@ -506,7 +503,7 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug, tim
       setupArgs(args, config)
       await testConfig.initializer(args)
     }
-    const result = await _process(config, test, { errorHandler, isTest: true, isProcess: true, isModule: false })
+    const result = await _process(config, test, { queryNumber: args.queryNumber, errorHandler, isTest: true, isProcess: true, isModule: false })
     result.query = test
     if (debug) {
       defaultInnerProcess(config, errorHandler, result)
@@ -626,6 +623,7 @@ const runTestsHelper = async (config, tests, failed, juicyBits) => {
     }
     const test = tests.shift()
     juicyBits.index = index
+    juicyBits.args.queryNumber = index
     const result = await runTest(config, test, juicyBits)
     if (result != null) {
       result.index = index
@@ -683,13 +681,6 @@ const saveTests = (config, testFile, testConfig) => {
   return saveTestsHelper(testFile, config, tests, tests.map((test) => test.query), testConfig)
 }
 
-/*
-const showExamples = (testFile) => {
-  const tests = JSON.parse(fs.readFileSync(testFile))
-  Object.keys(tests).forEach((test) => console.log(test))
-}
-*/
-
 const showInfo = (description, section, config) => {
   console.log(JSON.stringify(config.getInfo(), null, 2))
 }
@@ -803,7 +794,7 @@ const defaultErrorHandler = async (error) => {
   }
 
   if (error.query) {
-    console.log('query: ', error.query)
+    console.log(`query: #${error.queryNumber} ${error.query}`)
     doErrorExit = true
   }
 
@@ -895,6 +886,14 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
       console.log(`    inputs: ${JSON.stringify(inputs)} output: ${JSON.stringify(output)}  reason: ${reason}`)
     }
   }
+
+  if (responses.explain_associations_details) {
+    console.log(responses.explain_associations_trie)
+    for (const l of responses.explain_associations_details) {
+      console.log(l)
+    }
+  }
+
   // const objects = config.get('objects').namespaced[config.uuid]
   const actualGetObjects = (name) => {
     if (!name) {
@@ -1071,7 +1070,7 @@ const rebuildTemplate = async ({ config, instance, target, previousResultss, reb
           config.fragmentsBeingBuilt.push({ query: query.query, contexts: results.contexts })
         }
         accumulators.summaries = accumulators.summaries.concat(results.summaries)
-        accumulators.associations = accumulators.associations.concat(results.associations)
+        // accumulators.associations = accumulators.associations.concat(results.associations)
         accumulators.learned_contextual_priorities = accumulators.learned_contextual_priorities.concat(results.learned_contextual_priorities)
         await looper(configs)
       } catch (e) {
@@ -1107,7 +1106,7 @@ const rebuildTemplate = async ({ config, instance, target, previousResultss, reb
         await looper([])
       } else {
         try {
-          config.addInternal(_.cloneDeep(extraConfig), { handleCalculatedProps: true })
+          config.addInternal(_.cloneDeep(extraConfig), { handleCalculatedProps: true, addFirst: true })
         } catch (e) {
           const where = extraConfig.where ? ` ${extraConfig.where}` : ''
           throw new Error(`Error processing extra config${where}: ${e.stack}}`)
@@ -1129,8 +1128,9 @@ const rebuildTemplate = async ({ config, instance, target, previousResultss, reb
       associations.sort()
     }
     const stabilizeOutput = (template) => {
-      stabilizeAssociations(template.associations)
+      // stabilizeAssociations(template.associations)
       const stabilize = (results) => {
+        removeProp(results, (_, k) => k === "context_id")
         for (let i = 0; i < results.length; ++i) {
           const result = results[i]
           if (result.apply) {
@@ -1143,8 +1143,7 @@ const rebuildTemplate = async ({ config, instance, target, previousResultss, reb
             delete result.memory_free_percent
             delete result.logs
             delete result.version
-            result.hierarchy.sort()
-            stabilizeAssociations(result.associations)
+            // stabilizeAssociations(result.associations)
             result.learned_contextual_priorities = safeNoDups(result.learned_contextual_priorities)
           }
         }
@@ -1341,6 +1340,7 @@ const knowledgeModuleImpl = async ({
       parser.add_argument('-cl', '--checkForLoop', { nargs: '?', help: 'Check for loops in the priorities, Optional argument is list of operator keys to consider. For example [["banana", 0], ["food", 1]]' })
       parser.add_argument('-r', '--reset', { action: 'store_true', help: 'Get the server to bypass the cache and rebuild everything' })
       parser.add_argument('-q', '--query', { help: 'Run the specified query' })
+      parser.add_argument('-qn', '--queryNumber', { help: 'Run the query specified by the given index' })
       parser.add_argument('-f', '--filter', { help: 'for -pd only the data for the knowledge modules that start with this string will be shown' })
       parser.add_argument('-ip ', '--server', { help: 'Server to run against' })
       parser.add_argument('--trace', { action: 'store_true', help: 'Trace the semantics and generator calls.' })
@@ -1380,6 +1380,7 @@ const knowledgeModuleImpl = async ({
       parser.add_argument('-dcp', '--debugContextualPriority', { action: 'store_true', help: helpDebugContextualPriority })
       parser.add_argument('-db', '--debugBridge', { action: 'store_true', help: helpDebugBridge })
       parser.add_argument('-do', '--debugOperator', { action: 'store_true', help: helpDebugOperator })
+      parser.add_argument('-ea', '--explainAssociations', { action: 'store_true', help: 'The server will return information about the associations used to decide on the interpretation.' })
       parser.add_argument('-ep', '--explainPriorities', { action: 'store_true', help: 'The server will return all priorities including the generated one along with an explanation of there they came from' })
       parser.add_argument('-dic', '--debugIncludeConvolutions', { nargs: '?', help: 'When running with the --debugIncludeConvolutions flag the logs will include convolutions which are somewhat annoyingly verbose. Default is false' })
       parser.add_argument('-bc', '--bypassCache', { action: 'store_true', help: 'Bypass the cache on the server side and rebuild' })
@@ -1387,6 +1388,10 @@ const knowledgeModuleImpl = async ({
       const args = parser.parse_args()
       args.count = args.count || 1
 
+      if (args.queryNumber) {
+        args.query = test.contents[args.queryNumber].query
+      }
+
       if (args.rebuildTemplateFull) {
         args.rebuildTemplate = true
       }
@@ -1480,9 +1485,10 @@ const knowledgeModuleImpl = async ({
 
       if (args.deleteTest) {
         let tests = JSON.parse(runtime.fs.readFileSync(testConfig.name))
+        const nTests = tests.length
         tests = tests.filter((test) => test.query !== args.deleteTest)
         writeTestFile(testConfig.name, tests)
-        console.log(`Delete the test for "${args.deleteTest}"`)
+        console.log(`Deleted ${nTests - tests.length} tests for "${args.deleteTest}"`)
         return
       }
 
@@ -1513,6 +1519,10 @@ const knowledgeModuleImpl = async ({
         config.config.explain_priorities = true
       }
 
+      if (args.explainAssociations) {
+        config.config.explain_associations = true
+      }
+
       config.config.debugIncludeConvolutions = args.debugIncludeConvolutions || process.argv.includes('--debugIncludeConvolutions') || process.argv.includes('-dic')
 
       let configPrinted = false
@@ -1532,7 +1542,7 @@ const knowledgeModuleImpl = async ({
             console.log('Test queries')
             let counter = 0
             for (const test of config.tests) {
-              console.log(`${counter} - ${test.query}`)
+              console.log(`#${counter} - ${test.query}`)
               counter += 1
             }
           }
@@ -1545,7 +1555,10 @@ const knowledgeModuleImpl = async ({
           if (hasArg('c')) {
             const { data } = setupProcessB({ config })
             console.log('Config as sent to server')
-            console.log(JSON.stringify(data, null, 2))
+            const toPrint = { ...data }
+            toPrint['priorities'] = toPrint['priorities'].map((value) => JSON.stringify(value).replace(/\\"/g, '"'))
+            toPrint['associations']['positive'] = toPrint['associations']['positive'].map((value) => JSON.stringify(value).replace(/\\"/g, '"'))
+            console.log(JSON.stringify(toPrint, null, 2).replace(/\\"/g, '"'))
           }
           if (hasArg('l')) {
             console.log('Module load ordering')
@@ -1639,12 +1652,7 @@ const knowledgeModuleImpl = async ({
           }
 
           if (hasArg('s')) {
-            const easyToRead = _.cloneDeep(config.config.semantics)
-            for (const semantic of easyToRead) {
-              semantic.match = semantic.match.toString()
-              semantic.apply = semantic.apply.toString()
-            }
-            console.dir(easyToRead)
+            config.printSemantics()
           }
         }
       }
@@ -1761,7 +1769,7 @@ const knowledgeModuleImpl = async ({
                 if (hasError) {
                   console.log('**************************** ERRORS ************************')
                   for (const result of results) {
-                    console.log('Utterance: ', result.utterance)
+                    console.log(`Utterance: #${result.index} ${result.utterance}`)
                     if (!result.hasError) {
                       continue
                     }
@@ -1777,7 +1785,7 @@ const knowledgeModuleImpl = async ({
                         newError = true
                         headerShown = true
                         if (args.vimdiff) {
-                          vimdiff(actual, expected, `"${label} - ${result.index} ${result.utterance}"`)
+                          vimdiff(actual, expected, `"${label} - #${result.index} ${result.utterance}"`)
                         }
                         result.hasError = true
                       }
@@ -1920,12 +1928,17 @@ const knowledgeModuleImpl = async ({
                     console.log('use -v arg to write files expected.json and actual.json in the current directory for detailed comparison. Or do -s and then git diff the changes.')
                     // console.log(JSON.stringify(contexts))
                     let errorCount = 0
+                    const failedTests = []
                     for (const result of results) {
                       if (result.hasError) {
-                        console.log(`FAILED ${result.utterance}`)
+                        console.log(`FAILED #${result.index} ${result.utterance}`)
+                        failedTests.push({ index: result.index, utterance: result.utterance })
                         errorCount += 1
                       }
                     }
+                    const failedFile = './FAILED.json'
+                    console.log(`Failed tests are written to ${failedFile}`)
+                    runtime.fs.writeFileSync(failedFile, JSON.stringify(failedTests, 0, 2))
                     console.log(`**************************** THERE WERE ${errorCount} TEST FAILURES ************************`)
                   }
                 }
@@ -1984,7 +1997,7 @@ const knowledgeModuleImpl = async ({
           global.beforeObjects = _.cloneDeep(objects)
         }
         try {
-          await processResults(_process(config, args.query, { commandLineArgs: args, isProcess, isModule: !isProcess, dontAddAssociations: args.dontAddAssociations, writeTests: args.save || args.saveDeveloper, saveDeveloper: args.saveDeveloper, testConfig, testsFN: test }))
+          await processResults(_process(config, args.query, { commandLineArgs: args, isProcess, logs: [], isModule: !isProcess, dontAddAssociations: args.dontAddAssociations, writeTests: args.save || args.saveDeveloper, saveDeveloper: args.saveDeveloper, testConfig, testsFN: test }))
         } catch (error) {
           printConfig()
           console.log('Error', error)

package/index.js

@@ -1,7 +1,7 @@
 const { Semantics, Semantic } = require('./src/semantics')
 const { Generators, Generator } = require('./src/generators')
 const { Config } = require('./src/config')
-const { w, where, OverrideCheck } = require('./src/helpers')
+const { w, where, OverrideCheck, setByPath, getByPath, removeProp } = require('./src/helpers')
 const Digraph = require('./src/digraph')
 const client = require('./client')
 const flattens = require('./src/flatten')
@@ -16,6 +16,8 @@ module.exports = {
   runTests: client.runTests,
   knowledgeModule: client.knowledgeModule,
   ensureTestFile: client.ensureTestFile,
+  setByPath,
+  getByPath,
   where,
   stableId: client.stableId,
   w,
@@ -30,4 +32,5 @@ module.exports = {
   unflatten: unflatten.unflatten,
   debug,
   OverrideCheck,
+  removeProp,
 }

package/package.json

@@ -73,6 +73,6 @@
     "scriptjs": "^2.5.9",
     "uuid": "^8.3.2"
   },
-  "version": "9.7.1-beta.9",
+  "version": "9.8.0-beta.0",
   "license": "UNLICENSED"
 }

package/src/config.js

@@ -563,7 +563,7 @@ const handleCalculatedProps = (baseConfig, moreConfig, { addFirst, uuid } = {})
         'generatorr', 
         'generators', 
         'id', 
-        'init', 
+        'initial', 
         'inverted', 
         'isA', 
         'level', 
@@ -981,6 +981,15 @@ class Config {
     return config_toServer(config)
   }
 
+  printSemantics() {
+    const easyToRead = _.cloneDeep(this.config.semantics)
+    for (const semantic of easyToRead) {
+      semantic.match = semantic.match.toString()
+      semantic.apply = semantic.apply.toString()
+    }
+    console.dir(easyToRead)
+  }
+
   async run(handler) {
     return configHelpers.run(this, handler)
   }
@@ -1291,6 +1300,8 @@ class Config {
       } else {
         return fi
       }
+    } else {
+      throw new Error(`The fragment for '${query} was not found.`)
     }
   }
 
@@ -1681,6 +1692,7 @@ class Config {
         return
       }
     }
+
     if (global.transitoryMode) {
       def.transitoryMode = true
     }
@@ -3382,6 +3394,10 @@ class Config {
     let duplicated = new Set()
     const seen = new Set()
     for (const bridge of this.config.bridges) {
+      if (typeof bridge.id !== 'string') {
+        throw new Error('Config.addBridge: expected the bridge id to be a non-empty string.')
+      }
+
       const id = `${bridge.id}/${bridge.level} (namespace: ${bridge.uuid || this.uuid})`
       if (seen.has(id)) {
         duplicated.add(id)

package/src/configHelpers.js

@@ -165,10 +165,14 @@ const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
   }
   args.contextHierarchy = new ContextHierarchy()
   args.namespaced = {
-    get: (km, context, property) => {
+    get: (km, context, property, _default) => {
       context.namespaced ??= {}
       context.namespaced[km] ??= {}
-      return context.namespaced[km][property] 
+      const value = context.namespaced[km][property]
+      if (value == undefined) {
+        return _default
+      }
+      return value
     },
     set: (km, context, property, value) => {
       context.namespaced ??= {}
@@ -345,9 +349,11 @@ const processContext = async (context, { objects = {}, config, logs = [] }) => {
 const setupProcessB = ({ config, initializer, allowDelta = false, rebuildingTemplate = false } = {}) => {
   const key = config._key
   const data = Object.assign({ key, version: '3' }, { uuid: config._uuid })
+  /*
   if (rebuildingTemplate) {
     data.return_rtf_associations = true
   }
+  */
   if (allowDelta && config.allowDelta && config.hasDelta()) {
     // console.log('config', config)
     data.delta = config.delta()
@@ -405,7 +411,7 @@ const setupContexts = (rawContexts) => {
   return contexts
 }
 
-const processContextsB = async ({ config, calls, hierarchy, semantics, generators, json, isTest, isProcess, isModule, rebuildingTemplate, isInstance, instance, query, data, retries, url, commandLineArgs, forTemplate, contextIdCounter }) => {
+const processContextsB = async ({ config, calls, hierarchy, logs, semantics, generators, json, isTest, isProcess, isModule, rebuildingTemplate, isInstance, instance, query, data, retries, url, commandLineArgs, forTemplate, contextIdCounter }) => {
   // TODO fix this name to contextsPrime
   const contextsPrime = []
   const generatedPrime = []
@@ -417,10 +423,7 @@ const processContextsB = async ({ config, calls, hierarchy, semantics, generator
 
   const objects = config.get('objects')
   const args = { objects, isResponse: true, response: json, isTest, isInstance, getObjects: getObjects(objects), instance, contexts, isProcess, isModule, calls }
-  if (!json.logs) {
-    json.logs = []
-  }
-  setupArgs(args, config, json.logs, hierarchy)
+  setupArgs(args, config, logs, hierarchy)
   const toDo = [...contexts]
   args.insert = (context) => toDo.unshift(context)
   let overlap, lastRange
@@ -438,7 +441,7 @@ const processContextsB = async ({ config, calls, hierarchy, semantics, generator
       }
       const generateParenthesized = isTest || (commandLineArgs && commandLineArgs.save)
       if (!config.get('skipSemantics')) {
-        const semantics = config.getSemantics(json.logs)
+        const semantics = config.getSemantics(logs)
         try {
           contextPrime = await semantics.apply(args, context)
           // contextPrime.greg = 'yes'
@@ -481,11 +484,11 @@ const processContextsB = async ({ config, calls, hierarchy, semantics, generator
         // noop
       } else {
         let assumed = { isResponse: true }
-        const generated = contextPrime.isResponse ? await config.getGenerators(json.logs).apply({ ...args, assumed }, contextPrime, assumed) : ''
+        const generated = contextPrime.isResponse ? await config.getGenerators(logs).apply({ ...args, assumed }, contextPrime, assumed) : ''
         let generatedParenthesized = []
         if (generateParenthesized) {
           config.setParenthesized(true)
-          generatedParenthesized = contextPrime.isResponse ? await config.getGenerators(json.logs).apply({ ...args, assumed }, contextPrime, assumed) : ''
+          generatedParenthesized = contextPrime.isResponse ? await config.getGenerators(logs).apply({ ...args, assumed }, contextPrime, assumed) : ''
           config.setParenthesized(false)
         }
         // assumed = { paraphrase: true, response: false };
@@ -493,11 +496,11 @@ const processContextsB = async ({ config, calls, hierarchy, semantics, generator
         if (generateParenthesized) {
           config.setParenthesized(false)
         }
-        const paraphrases = await config.getGenerators(json.logs).apply({ ...args, assumed }, contextPrime, assumed)
+        const paraphrases = await config.getGenerators(logs).apply({ ...args, assumed }, contextPrime, assumed)
         let paraphrasesParenthesized = []
         if (generateParenthesized) {
           config.setParenthesized(true)
-          paraphrasesParenthesized = await config.getGenerators(json.logs).apply({ ...args, assumed }, contextPrime, assumed)
+          paraphrasesParenthesized = await config.getGenerators(logs).apply({ ...args, assumed }, contextPrime, assumed)
           config.setParenthesized(false)
         }
         contextsPrime.push(contextPrime)
@@ -526,9 +529,9 @@ const processContextsB = async ({ config, calls, hierarchy, semantics, generator
       }
       e.context = contextPrime
       if (e.logs) {
-        e.logs = e.logs.concat(json.logs)
+        e.logs = e.logs.concat(logs)
       } else {
-        e.logs = json.logs
+        e.logs = logs
       }
       e.metadata = json.metadata
       if (json.trace) {
@@ -545,10 +548,12 @@ const loadInstance = async (config, instance) => {
   const transitoryMode = global.transitoryMode
   global.transitoryMode = false
 
+  /*
   const rl = instance.resultss.length
   if (rl > 0) {
     config.addAssociations(instance.resultss[instance.resultss.length - 1].rtf_associations || [])
   }
+  */
   /*
   TODO needs updating if still wanted
   if (instance && (instance.associations || instance.learned_contextual_priorities)) {

package/src/digraph.js

@@ -73,11 +73,25 @@ class Digraph {
     this._edges = edges
   }
 
-  /*
-  set edges(edges) {
-    this._edges = edges.map( toA )
+  tos(from) {
+    const tos = []
+    for (const edge of this._edges) {
+      if (edge[0] == from) {
+        tos.push(edge[1])
+      }
+    }
+    return tos
+  }
+
+  froms(to) {
+    const froms = []
+    for (const edge of this._edges) {
+      if (edge[1] == to) {
+        froms.push(edge[0])
+      }
+    }
+    return froms
   }
-*/
 
   acdcs (s, from, to) {
     if (!s) {
@@ -170,18 +184,6 @@ class Digraph {
     return this.minima(common)
   }
 
-  /*
-  maxima (nodes) {
-    const maxima = new Set(nodes)
-    const descendants = new Set([])
-    nodes.forEach((node) => {
-      this.descendants(node).forEach((n) => descendants.add(n))
-    })
-    descendants.forEach((n) => maxima.delete(n))
-    return maxima
-  }
-  */
-
   add (child, parent) {
     this._edges.push([child, parent])
   }

package/src/digraph_internal.js

@@ -39,6 +39,26 @@ class DigraphInternal {
     this._edges = edges
   }
 
+  tos(from) {
+    const tos = []
+    for (const edge of this._edges) {
+      if (edge[0] == from) {
+        tos.push(edge[1])
+      }
+    }
+    return tos
+  }
+
+  froms(to) {
+    const froms = []
+    for (const edge of this._edges) {
+      if (edge[1] == to) {
+        froms.push(edge[0])
+      }
+    }
+    return froms
+  }
+
   get length () {
     return this._edges.length
   }
@@ -137,18 +157,6 @@ class DigraphInternal {
     return this.minima(common)
   }
 
-  /*
-  maxima (nodes) {
-    const maxima = new Set(nodes)
-    const descendants = new Set([])
-    nodes.forEach((node) => {
-      this.descendants(node).forEach((n) => descendants.add(n))
-    })
-    descendants.forEach((n) => maxima.delete(n))
-    return maxima
-  }
-  */
-
   add (child, parent) {
     this._edges.push([child, parent])
   }

package/src/flatten.js

@@ -76,6 +76,9 @@ const flatten = (markers, value) => {
   if (value.flatten === false) {
     return [[value], false]
   }
+
+  const flatten_ignore = value.flatten_ignore || []
+
   const marker = value.marker
   let properties = value
 
@@ -105,8 +108,9 @@ const flatten = (markers, value) => {
   for (const key in properties) {
     let wf = false
     let values
-    // if isinstance(properties[key], Context) {
-    if (isObject(properties[key])) {
+    if (flatten_ignore.includes(key)) {
+      values = [properties[key]]
+    } else if (isObject(properties[key])) {
       const context = properties[key];
       [values, wf] = flatten(markers, context)
     // } else if (isinstance(properties[key], list)) {

package/src/fragments.js

@@ -23,7 +23,6 @@ function createChangeTracker(obj) {
   };
 }
 
-
 function fragmentInstantiator (args, contexts) {
   return new Object({ 
     contexts: () => {

package/src/generators.js

@@ -125,11 +125,10 @@ class Generator {
       args.g = args.gp
     }
     return await debug.hitScoped(args.callId, async () => {
-      if ((options.debug || {}).apply ||
-          callId === this.callId) {
+      if ((options.debug || {}).apply || process.env.DEBUG_CALLID == args.callId || callId === this.callId) {
         debugger // eslint-disable-line no-debugger
       }
-      return await this._apply(args)
+      return await this._apply(args)  // DEBUG: This is the line you want to step into
     })
   }
 }
@@ -185,12 +184,22 @@ class Generators {
     let generated = ''
     let applied = false
     const stack = args.calls.push()
+    const pOrRTag = assumed.isResponse ? "(RESPONSE)" : "(PARAPHRASE)"
+    const finallyList = []; // called after the generator is processed
+    args._finally = (call) => finallyList.unshift(call)
     for (let igenerator = 0; igenerator < this.generators.length; ++igenerator) {
       const generator = this.generators[igenerator]
       if (await generator.matches(args, objects, context, hierarchy, config, options)) {
         const log = (message) => { this.logs.push(message) }
         // this.logs.push(`Generators: applied ${generator.toString()}\n to\n ${stringify(context)}`)
         let errorMessage = 'The apply function did not return a value'
+        if (process.env.TPMKMS_TRACE) {
+          console.error("TPMKMS_TRACE", debug.counter("TPMKMS_TRACE"))
+          console.error(generator.toLabel())
+          console.error(generator.toString())
+          console.log(`To debug this use debug.breakAt('${args.calls.current()}') or DEBUG_CALLID="${args.calls.current()}"`)
+        }
+
         try {
           generated = await generator.apply(args, objects, context, hierarchy, config, response, log, options)
         } catch (e) {
@@ -212,6 +221,7 @@ class Generators {
           const widths = Lines.addRemainder([10, 10])
           const lines = new Lines(widths)
           lines.setElement(0, 0, 'Generator')
+          lines.setElement(0, 1, `#${context.context_id || "?"}`)
           const source = `${generator.km}/#${generator.index}`
           lines.setElement(0, 2, `ERROR while applying (${source}) ${generator.toLabel()}`)
           lines.newRow()
@@ -225,7 +235,7 @@ class Generators {
           lines.setElement(0, 2, stack)
           lines.newRow()
           lines.setElement(0, 1, 'DEBUG')
-          lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}')`)
+          lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}') or DEBUG_CALLID="${args.calls.current()}"`)
           lines.newRow()
           lines.setElement(0, 1, 'ERROR')
           lines.setElement(0, 2, errorMessage)
@@ -240,6 +250,7 @@ class Generators {
           const widths = Lines.addRemainder([10, 10])
           const lines = new Lines(widths)
           lines.setElement(0, 0, 'Generator')
+          lines.setElement(0, 1, `#${context.context_id || "?"}`)
           if (generator.index > -1 && generator.km) {
             // lines.setElement(0, 2, `KM '${generator.km}' ordinal: ${generator.index}`)
             lines.setElement(0, 2, generator.toLabel())
@@ -251,7 +262,7 @@ class Generators {
           lines.setElement(0, 2, generator.toString())
           lines.newRow()
           lines.setElement(0, 1, 'RESULT')
-          lines.setElement(0, 2, generated)
+          lines.setElement(0, 2, generated + " " + pOrRTag)
           lines.newRow()
           lines.setElement(0, 1, 'STACK')
           lines.setElement(0, 2, stack)
@@ -268,11 +279,15 @@ class Generators {
         break
       }
     }
+    for (const f of finallyList) {
+      await f()
+    }
     args.calls.pop()
     if (!applied && config.getDebug()) {
       const widths = Lines.addRemainder([10, 10])
       const lines = new Lines(widths)
       lines.setElement(0, 0, 'Generator')
+      lines.setElement(0, 1, `#${context.context_id || "?"}`)
       lines.setElement(0, 2, 'No generator applied')
       lines.newRow()
       lines.setElement(0, 1, 'STACK')

package/src/helpers.js

@@ -544,52 +544,77 @@ const stableId = (tag) => {
 }
 
 function getByPath(obj, path, defaultValue) {
+  const segments = Array.isArray(path) ? path : [path];
+  const keys = segments.flatMap((segment) => {
+    if (typeof segment !== "string") return [segment];
+    return segment
+      .replace(/\[(\d+)\]/g, ".$1")
+      .split(".")
+      .filter((key) => key !== "");
+  });
+
   let current = obj;
-  for (const key of path) {
+  for (const key of keys) {
     if (current === null || current === undefined) return defaultValue;
-    if (typeof current !== 'object') return defaultValue;
+    if (typeof current !== "object") return defaultValue;
     current = current[key];
   }
   return current === undefined ? defaultValue : current;
 }
 
+function normalizePath(path) {
+  const segments = Array.isArray(path) ? path : [path];
+  return segments.flatMap((segment) => {
+    if (typeof segment !== "string") return [segment];
+    return segment
+      .replace(/\[(\d+)\]/g, ".$1")
+      .split(".")
+      .filter((key) => key !== "");
+  });
+}
+
+function isArrayIndex(key) {
+  if (typeof key === "number") return Number.isInteger(key) && key >= 0;
+  return key !== "" && String(key >>> 0) === String(key);
+}
+
 /**
- * Set a value in an object by path array.
+ * Set a value in an object by path (string or array).
  * Automatically creates missing objects {} or arrays [] as needed.
  *
  * @param {Object} obj - The root object to modify
- * @param {Array<string|number>} path - Array of keys/indices
+ * @param {string|Array<string|number>} path - Path keys/indices
  * @param {*} value - Value to set
  * @returns {*} The set value (for chaining)
  */
 function setByPath(obj, path, value) {
-  if (!Array.isArray(path) || path.length === 0) {
-    throw new Error('Path must be a non-empty array');
+  const keys = normalizePath(path);
+
+  if (keys.length === 0) {
+    throw new Error("Path must be a non-empty string or array");
+  }
+
+  if (obj == null || typeof obj !== "object") {
+    throw new Error("Target must be an object or array");
   }
 
   let current = obj;
 
-  for (let i = 0; i < path.length; i++) {
-    const key = path[i];
-    const isLast = i === path.length - 1;
+  for (let i = 0; i < keys.length; i++) {
+    const key = keys[i];
+    const isLast = i === keys.length - 1;
 
     if (isLast) {
-      // Final step — just assign
       current[key] = value;
     } else {
-      // Not last — ensure next level exists
-      const nextKey = path[i + 1];
-
-      if (current[key] == null) {
-        // Auto-create: array if next key is number, otherwise object
-        current[key] = typeof nextKey === 'number' || String(nextKey >>> 0) === nextKey
-          ? []
-          : {};
-      } else if (Array.isArray(current[key]) && typeof nextKey !== 'number') {
-        // Safety: if current is array but next key isn't a valid index → convert to object
+      const nextKey = keys[i + 1];
+      const nextIsIndex = isArrayIndex(nextKey);
+
+      if (current[key] == null || typeof current[key] !== "object") {
+        current[key] = nextIsIndex ? [] : {};
+      } else if (Array.isArray(current[key]) && !nextIsIndex) {
         current[key] = { ...current[key] };
-      } else if (!Array.isArray(current[key]) && typeof nextKey === 'number') {
-        // If next expects array but current is object → convert
+      } else if (!Array.isArray(current[key]) && nextIsIndex) {
         current[key] = Object.values(current[key]);
       }
 
@@ -602,7 +627,60 @@ function setByPath(obj, path, value) {
 
 const helpDebugHierarchy = 'In order to get a debug break when a specific hierarchy is created set the DEBUG_HIERARCHY environment variable to the JSON of the child-parent pair to break on. For example DEBUG_HIERARCHY=\'["cat", "mammel"]\''
 
+// removeProp.js
+function removeProp(obj, testFn, { maxDepth = Infinity, seen = new WeakSet() } = {}) {
+  if (!obj || typeof obj !== 'object' || maxDepth <= 0) return obj;
+  if (seen.has(obj)) return obj;
+  seen.add(obj);
+
+  if (Array.isArray(obj)) {
+    // ---- ARRAY: process each element (but don't remove elements unless testFn says so)
+    let writeIdx = 0;
+    for (let i = 0; i < obj.length; i++) {
+      const element = obj[i];
+      const shouldRemoveElement = testFn(element, i, obj);
+
+      if (shouldRemoveElement) {
+        // Remove the whole array element
+        if (element && typeof element === 'object') {
+          // Still walk inside it in case testFn wants side effects
+          removeProp(element, testFn, { maxDepth: maxDepth - 1, seen });
+        }
+        // Skip writing it back
+      } else {
+        // Keep element, but walk into it to remove inner props
+        removeProp(element, testFn, { maxDepth: maxDepth - 1, seen });
+        if (writeIdx !== i) {
+          obj[writeIdx] = element;
+        }
+        writeIdx++;
+      }
+    }
+    obj.length = writeIdx;
+    return obj;
+  }
+
+  // ---- OBJECT: iterate over own keys
+  const keys = Reflect.ownKeys(obj);
+  for (const key of keys) {
+    const val = obj[key];
+    const shouldRemove = testFn(val, key, obj);
+
+    if (shouldRemove) {
+      delete obj[key];
+      if (val && typeof val === 'object') {
+        removeProp(val, testFn, { maxDepth: maxDepth - 1, seen });
+      }
+    } else {
+      removeProp(val, testFn, { maxDepth: maxDepth - 1, seen });
+    }
+  }
+
+  return obj;
+}
+
 module.exports = {
+  removeProp,
   helpDebugHierarchy,
 
   stableId,

package/src/project2.js

@@ -101,6 +101,10 @@ function project(source, filters, path=[]) {
     } else if (prop.property && source.hasOwnProperty(prop.property)) {
       // If the property is an object and not null, recursively project it
       if (typeof source[prop.property] === 'object' && source[prop.property] !== null) {
+        if (prop.all) {
+          result[prop.property] = source[prop.property]
+          return
+        }
         result[prop.property] = {}
         const instantiatedCheck = []
         for (const check of prop.check) {

package/src/semantics.js

@@ -77,10 +77,11 @@ class Semantic {
     this.fixUpArgs(args, context)
     return await debug.hitScoped(args.callId, async () => {
       const matches = await this.matcher(args)
-      if (matches && (options.debug || {}).match || args.callId === this.callId) {
-        // next line is the matcher
-        debugger // eslint-disable-line no-debugger
-        await this.matcher(args)
+      if (matches) {
+        if ((options.debug || {}).match || args.callId === this.callId || process.env.DEBUG_CALLID == args.callId) {
+          debugger // eslint-disable-line no-debugger
+          await this.matcher(args) // DEBUG: This is the line you step into
+        }
       }
       return matches
     })
@@ -100,8 +101,8 @@ class Semantic {
     this.fixUpArgs(args, contextPrime)
 
     return await debug.hitScoped(args.callId, async () => {
-      if ((options.debug || {}).apply || args.callId === this.callId) {
-        debugger // eslint-disable-line no-debugger
+      if ((options.debug || {}).apply || args.callId === this.callId || process.env.DEBUG_CALLID == args.callId) {
+        debugger // eslint-disable-line no-debugger  // STEP ABOUT THREE TIMES TO THE DEBUG LINE
       }
       if (args.breakOnSemantics) {
         debugger // eslint-disable-line no-debugger
@@ -109,7 +110,7 @@ class Semantic {
 
       try {
         args.stack.push(this)
-        await this._apply(args)
+        await this._apply(args) // DEBUG: This is the line you want to step into
         args.stack.pop()
       } catch( e ) {
         args.stack.pop()
@@ -192,6 +193,8 @@ class Semantics {
     let seenQuestion = false
     const deferred = []
     args.log = (message) => { this.logs.push(message) }
+    const finallyList = []; // called after the semantic is processed
+    args._finally = (call) => finallyList.unshift(call)
     for (const isemantic in this.semantics) {
       const semantic = this.semantics[isemantic]
       // only one question at a time
@@ -252,13 +255,14 @@ class Semantics {
           const widths = Lines.addRemainder([10, 10])
           const lines = new Lines(widths)
           lines.setElement(0, 0, 'Semantic')
+          lines.setElement(0, 1, `#${context.context_id || "?"}`)
           const source = `${semantic.km}/#${semantic.index}`
           lines.setElement(0, 2, `ERROR while applying (${source}) ${semantic.toLabel()}`)
           lines.newRow()
           lines.setElement(0, 2, semantic.toString())
           lines.newRow()
           lines.setElement(0, 1, 'DEBUG')
-          lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}')`)
+          lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}') in the code or DEBUG_CALLID="${args.calls.current()}"`)
           lines.newRow()
           lines.setElement(0, 1, 'TO')
           lines.setElement(0, 2, `context_id: ${context.context_id}`)
@@ -282,6 +286,7 @@ class Semantics {
           const widths = Lines.addRemainder([10, 10])
           const lines = new Lines(widths)
           lines.setElement(0, 0, 'Semantic')
+          lines.setElement(0, 1, `#${context.context_id || "?"}`)
           if (semantic.index > -1 && semantic.km) {
             // lines.setElement(0, 2, `KM '${semantic.km}' ordinal: ${semantic.index} ${ semantic.notes ? semantic.nodes : '' }`)
             lines.setElement(0, 2, semantic.toLabel())
@@ -293,7 +298,7 @@ class Semantics {
           lines.setElement(0, 2, semantic.toString())
           lines.newRow()
           lines.setElement(0, 1, 'DEBUG')
-          lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}')`)
+          lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}') in the code or DEBUG_CALLID="${args.calls.current()}"`)
           lines.newRow()
           lines.setElement(0, 1, 'TO')
           lines.setElement(0, 2, `context_id: ${context.context_id}`)
@@ -324,11 +329,15 @@ class Semantics {
       }
       counter += 1
     }
+    for (const f of finallyList) {
+      await f()
+    }
     args.calls.pop()
     if (!applied && debug) {
       const widths = Lines.addRemainder([10, 10])
       const lines = new Lines(widths)
       lines.setElement(0, 0, 'Semantic')
+      lines.setElement(0, 1, `#${context.context_id || "?"}`)
       lines.setElement(0, 2, 'No semantic applied')
       lines.newRow()
       lines.setElement(0, 1, 'STACK')