ekms 7.12.8-beta.5 → 7.12.8-beta.7

package/common/time.test.json

@@ -7,7 +7,7 @@
           0
         ],
         [
-          "hourUnits",
+          "number",
           0
         ]
       ]
@@ -21,17 +21,18 @@
         "determined": true,
         "hour": {
           "dead": true,
-          "hour": 2,
+          "instance": true,
           "level": 1,
-          "marker": "hourUnits",
+          "marker": "number",
           "range": {
             "end": 0,
             "start": 0
           },
           "text": "2",
           "types": [
-            "hourUnits"
+            "number"
           ],
+          "value": 2,
           "word": "2"
         },
         "level": 1,
@@ -53,7 +54,7 @@
         {
           "counter": 1,
           "op": [
-            "hourUnits",
+            "number",
             0
           ],
           "ops": [
@@ -62,7 +63,7 @@
               0
             ],
             [
-              "hourUnits",
+              "number",
               0
             ]
           ]
@@ -87,6 +88,7 @@
         "dialogues": "dialogues2",
         "gdefaults": "gdefaults2",
         "meta": "meta2",
+        "numbers": "numbers2",
         "pos": "pos2",
         "punctuation": "punctuation2",
         "sdefaults": "sdefaults2",
@@ -105,6 +107,8 @@
         },
         "meta2": {
         },
+        "numbers2": {
+        },
         "pos2": {
         },
         "punctuation2": {
@@ -131,17 +135,18 @@
             "determined": true,
             "hour": {
               "dead": true,
-              "hour": 2,
+              "instance": true,
               "level": 1,
-              "marker": "hourUnits",
+              "marker": "number",
               "range": {
                 "end": 0,
                 "start": 0
               },
               "text": "2",
               "types": [
-                "hourUnits"
+                "number"
               ],
+              "value": 2,
               "word": "2"
             },
             "level": 1,
@@ -156,7 +161,7 @@
           },
           "generatedParenthesized": "",
           "paraphrases": "2 pm",
-          "paraphrasesParenthesized": "(2 pm)",
+          "paraphrasesParenthesized": "((2) pm)",
           "responses": [
             ""
           ]
@@ -167,7 +172,7 @@
       "2 pm"
     ],
     "paraphrasesParenthesized": [
-      "(2 pm)"
+      "((2) pm)"
     ],
     "query": "2 pm",
     "responses": [
@@ -182,15 +187,15 @@
           0
         ],
         [
-          "hourUnits",
+          "info",
           0
         ],
         [
-          "info",
+          "is",
           0
         ],
         [
-          "is",
+          "number",
           0
         ],
         [
@@ -216,15 +221,15 @@
           0
         ],
         [
-          "hourUnits",
+          "info",
           0
         ],
         [
-          "info",
+          "is",
           0
         ],
         [
-          "is",
+          "number",
           0
         ],
         [
@@ -246,11 +251,11 @@
           0
         ],
         [
-          "hourUnits",
+          "is",
           0
         ],
         [
-          "is",
+          "number",
           0
         ],
         [
@@ -264,11 +269,11 @@
           0
         ],
         [
-          "hourUnits",
+          "is",
           0
         ],
         [
-          "is",
+          "number",
           0
         ],
         [
@@ -286,11 +291,11 @@
           0
         ],
         [
-          "hourUnits",
+          "is",
           0
         ],
         [
-          "is",
+          "number",
           0
         ],
         [
@@ -405,17 +410,18 @@
             "determined": true,
             "hour": {
               "dead": true,
-              "hour": 2,
+              "instance": true,
               "level": 1,
-              "marker": "hourUnits",
+              "marker": "number",
               "range": {
                 "end": 25,
                 "start": 25
               },
               "text": "2",
               "types": [
-                "hourUnits"
+                "number"
               ],
+              "value": 2,
               "word": "2"
             },
             "level": 1,
@@ -497,15 +503,15 @@
               0
             ],
             [
-              "hourUnits",
+              "info",
               0
             ],
             [
-              "info",
+              "is",
               0
             ],
             [
-              "is",
+              "number",
               0
             ],
             [
@@ -538,15 +544,15 @@
               0
             ],
             [
-              "hourUnits",
+              "info",
               0
             ],
             [
-              "info",
+              "is",
               0
             ],
             [
-              "is",
+              "number",
               0
             ],
             [
@@ -575,11 +581,11 @@
               0
             ],
             [
-              "hourUnits",
+              "is",
               0
             ],
             [
-              "is",
+              "number",
               0
             ],
             [
@@ -608,11 +614,11 @@
               0
             ],
             [
-              "hourUnits",
+              "is",
               0
             ],
             [
-              "is",
+              "number",
               0
             ],
             [
@@ -628,7 +634,7 @@
         {
           "counter": 5,
           "op": [
-            "hourUnits",
+            "number",
             0
           ],
           "ops": [
@@ -637,11 +643,11 @@
               0
             ],
             [
-              "hourUnits",
+              "is",
               0
             ],
             [
-              "is",
+              "number",
               0
             ],
             [
@@ -712,6 +718,7 @@
         "dialogues": "dialogues2",
         "gdefaults": "gdefaults2",
         "meta": "meta2",
+        "numbers": "numbers2",
         "pos": "pos2",
         "punctuation": "punctuation2",
         "sdefaults": "sdefaults2",
@@ -730,6 +737,8 @@
         },
         "meta2": {
         },
+        "numbers2": {
+        },
         "pos2": {
         },
         "punctuation2": {
@@ -793,17 +802,18 @@
                 "determined": true,
                 "hour": {
                   "dead": true,
-                  "hour": 2,
+                  "instance": true,
                   "level": 1,
-                  "marker": "hourUnits",
+                  "marker": "number",
                   "range": {
                     "end": 25,
                     "start": 25
                   },
                   "text": "2",
                   "types": [
-                    "hourUnits"
+                    "number"
                   ],
+                  "value": 2,
                   "word": "2"
                 },
                 "level": 1,
@@ -869,7 +879,7 @@
           },
           "generatedParenthesized": "",
           "paraphrases": "tell me when the time is 2 pm",
-          "paraphrasesParenthesized": "(tell (me) when ((the (time)) is (2 pm)))",
+          "paraphrasesParenthesized": "(tell (me) when ((the (time)) is ((2) pm)))",
           "responses": [
             ""
           ]
@@ -880,7 +890,7 @@
       "tell me when the time is 2 pm"
     ],
     "paraphrasesParenthesized": [
-      "(tell (me) when ((the (time)) is (2 pm)))"
+      "(tell (me) when ((the (time)) is ((2) pm)))"
     ],
     "query": "tell me when the time is 2 pm",
     "responses": [
@@ -1386,6 +1396,7 @@
         "dialogues": "dialogues2",
         "gdefaults": "gdefaults2",
         "meta": "meta2",
+        "numbers": "numbers2",
         "pos": "pos2",
         "punctuation": "punctuation2",
         "sdefaults": "sdefaults2",
@@ -1404,6 +1415,8 @@
         },
         "meta2": {
         },
+        "numbers2": {
+        },
         "pos2": {
         },
         "punctuation2": {
@@ -2122,6 +2135,7 @@
         "dialogues": "dialogues2",
         "gdefaults": "gdefaults2",
         "meta": "meta2",
+        "numbers": "numbers2",
         "pos": "pos2",
         "punctuation": "punctuation2",
         "sdefaults": "sdefaults2",
@@ -2140,6 +2154,8 @@
         },
         "meta2": {
         },
+        "numbers2": {
+        },
         "pos2": {
         },
         "punctuation2": {
@@ -2531,6 +2547,7 @@
         "dialogues": "dialogues2",
         "gdefaults": "gdefaults2",
         "meta": "meta2",
+        "numbers": "numbers2",
         "pos": "pos2",
         "punctuation": "punctuation2",
         "sdefaults": "sdefaults2",
@@ -2549,6 +2566,8 @@
         },
         "meta2": {
         },
+        "numbers2": {
+        },
         "pos2": {
         },
         "punctuation2": {
@@ -2905,6 +2924,7 @@
         "dialogues": "dialogues2",
         "gdefaults": "gdefaults2",
         "meta": "meta2",
+        "numbers": "numbers2",
         "pos": "pos2",
         "punctuation": "punctuation2",
         "sdefaults": "sdefaults2",
@@ -2923,6 +2943,8 @@
         },
         "meta2": {
         },
+        "numbers2": {
+        },
         "pos2": {
         },
         "punctuation2": {

package/common/tokenize.js

@@ -0,0 +1,46 @@
+const { Config, knowledgeModule, where } = require('./runtime').theprogrammablemind
+const { defaultContextCheck } = require('./helpers')
+const tokenize_tests = require('./tokenize.test.json')
+
+let configStruct = {
+  name: 'tokenize',
+  operators: [
+    "([unknown])",
+  ],
+  bridges: [
+    { id: "unknown", level: 0, bridge: "{ ...next(operator), unknown: true, dead: true }" },
+    { id: "unknown", level: 1, bridge: "{ ...next(operator) }" },
+  ],
+  words: {
+    patterns: [
+      { pattern: [{ type: 'space' }, { repeat: true }], defs: [ { remove: true } ] },
+      { pattern: [{ type: 'alphanumeric' }, { repeat: true }], defs: [ { id: 'unknown', initial: "{ value: text, unknown: true }" } ] },
+    ],
+    hierarchy: [
+      { child: ' ', parent: 'space' },
+      ..."0123456789".split("").map((digit) => { return { child: digit, parent: 'digit' } }),
+      { child: 'lower', parent: 'letter' }, 
+      { child: 'upper', parent: 'letter' },
+      ...'abcdefghijklmnopqrstuvwxyz'.split("").map((letter) => { return { child: letter, parent: 'lower' } }),
+      ...'ABCDEFGHIJKLMNOPQRSTUVWXYZ'.split("").map((letter) => { return { child: letter, parent: 'upper' } }),
+      { child: 'letter', parent: 'alphanumeric' },
+      { child: 'digit', parent: 'alphanumeric' },
+      ...`~!@#$%^&*()_+-=[]{}\\|;:,<>/?'"`.split("").map((punctuation) => { return { child: punctuation, parent: 'punctuation' } }),
+    ],
+  },
+};
+
+createConfig = () => new Config(configStruct, module)
+
+knowledgeModule( { 
+  module,
+  createConfig,
+  description: 'tokenize',
+  test: {
+    name: './tokenize.test.json',
+    contents: tokenize_tests,
+    checks: {
+            context: defaultContextCheck,
+    },
+  },
+})

package/common/tokenize.test.json

@@ -0,0 +1,2 @@
+[
+]

package/common/ui.instance.json

@@ -1,5 +1,5 @@
 {
-  "queries": [],
+  "configs": [],
   "resultss": [],
   "fragments": [
     {