@fraczak/k 2.1.6 → 3.0.2

package/chatGPT.md~

@@ -0,0 +1,119 @@
+Thank you for the clarification. Let's revise the explanation to correctly describe the projection from union types as a check.
+
+### Key Concepts and Constructs of Kernel/Core
+
+1. **Program Structure**:
+   - Consists of a possibly empty list of statements, each ending with a semicolon (`;`), followed by an expression.
+   - Statements can be:
+     - **Code Definitions**: Define data types and start with a dollar sign (`$`).
+     - **Relation Definitions**: Define partial functions for data transformations.
+
+2. **Code Definitions (Data Types)**:
+   - **Union Types**: Defined using angle brackets (`<...>`).
+     ```kernelcore
+     $ bool = <{} true, {} false>;
+     ```
+   - **Product Types**: Defined using curly braces (`{...}`).
+     ```kernelcore
+     $ pair_of_bools = {bool x, bool y};
+     ```
+
+3. **Relation Definitions (Partial Functions)**:
+   - **Projection (Field Extraction)**: Partial functions that extract fields from product types or check for specific variants in union types.
+     ```kernelcore
+     // Extract the `x` field from a pair_of_bools (product type)
+     extract_x = $pair_of_bools .x;
+     
+     // Extract the `y` field from a pair_of_bools (product type)
+     extract_y = $pair_of_bools .y;
+     
+     // Check if a bool is the `true` variant (union type)
+     check_true = $bool .true;
+     
+     // Check if a bool is the `false` variant (union type)
+     check_false = $bool .false;
+     ```
+   - **Functional Composition**: Sequential application of partial functions.
+     ```kernelcore
+     is_x_true = $pair_of_bools .x .true;
+     ```
+   - **Parallel (Product) Composition**: Combine multiple partial functions to construct a new product type.
+     ```kernelcore
+     extract_both = $pair_of_bools {.x x_value, .y y_value};
+     ```
+   - **Concurrent (Union) Composition**: Try different functions in sequence, returning the first successful result.
+     ```kernelcore
+     is_any_true = $pair_of_bools <.x .true true_bool, .y .true true_bool, false_bool>;
+     ```
+
+4. **Constants**:
+   - Constants are partial functions that ignore their input.
+     ```kernelcore
+     true_bool = {{} true};
+     false_bool = {{} false};
+     ```
+
+5. **Names and Labels**:
+   - Names and labels are alphanumerical sequences of ASCII characters, following the convention `[a-zA-Z_][a-zA-Z0-9_]*`.
+
+6. **Final Expression**:
+   - A final expression evaluates usage examples and collects results in a single product.
+     ```kernelcore
+     {pair1 is_x_true result1, 
+      pair2 is_x_true result2,
+      pair1 extract_both both_values, 
+      pair1 is_any_true any_true1, 
+      pair2 is_any_true any_true2, 
+      pair3 is_any_true any_true3}
+     ```
+
+### Complete Example Program
+
+```kernelcore
+// Define a boolean type as a union of true and false
+$ bool = <{} true, {} false>;
+
+// Define a pair of booleans as a product of two booleans
+$ pair_of_bools = {bool x, bool y};
+
+// Projection (Field Extraction)
+// Extract the `x` field from a pair_of_bools (product type)
+extract_x = $pair_of_bools .x;
+
+// Extract the `y` field from a pair_of_bools (product type)
+extract_y = $pair_of_bools .y;
+
+// Check if a bool is the `true` variant (union type)
+check_true = $bool .true;
+
+// Check if a bool is the `false` variant (union type)
+check_false = $bool .false;
+
+// Check if the `x` field is `true`
+is_x_true = $pair_of_bools .x .true;
+
+// Extract both `x` and `y` fields into a new product
+extract_both = $pair_of_bools {.x x_value, .y y_value};
+
+// Check if either `x` or `y` is `true`
+is_any_true = $pair_of_bools <.x .true true_bool, .y .true true_bool, false_bool>;
+
+// Define constants
+true_bool = {{} true};
+false_bool = {{} false};
+
+// Create instances of `pair_of_bools`
+pair1 = {true_bool x, true_bool y};
+pair2 = {true_bool x, false_bool y};
+pair3 = {false_bool x, false_bool y};
+
+// Final expression to evaluate usage examples
+{pair1 is_x_true result1, 
+ pair2 is_x_true result2,
+ pair1 extract_both both_values, 
+ pair1 is_any_true any_true1, 
+ pair2 is_any_true any_true2, 
+ pair3 is_any_true any_true3}
+```
+
+This updated introduction and complete example program now correctly describe the projection from union types as checks, providing a comprehensive overview of kernel/core. If there are any further questions or a

package/code-resolution.md

@@ -0,0 +1,121 @@
+# Pattern and its graph representation
+
+Intuitively, a _pattern_ is an adhock description of a set of codes
+by providing some characteristics of the codes. Those characteristics are:
+
+  1. Is this product vs union?
+  2. The code has to have field (label) leading to a type or another parretn
+  
+Code patterns is a graph with:
+
+  1. edges labeled by `fields`
+  2. nodes labeled by one of the:
+
+    1. `/*/` - open unkown
+    2. `{*}` - open product
+    3. `<*>` - open union
+    4. `//` - close uknown
+    5. `{}` - close product
+    6. `<>` - close union
+
+A graph without "open" and "unknown" nodes is "code".
+
+Actually every node in such a graph corresponds to a code pattern.
+
+## Pattern Graph
+
+Example:
+
+    $nat = < {} _, nat 0, nat 1>;
+    rlz = $nat <.0._ {{{} _} 0}, .0 rlz, ()>;
+
+We get `codes`:
+
+    id    |       code
+    ------|----------------------
+    '{}'  | {}
+    'nat' | <nat 0, nat 1, {} _>
+
+and the (singleton) forest of expressons:
+
+    id    | ref-name | rel
+    ------|----------|--------------
+    0     | 'rlz'    | ($nat < (.0 ._ {{{} _} 0}), (.0 rlz), ()>)
+
+### Initialize pattern graph
+
+1. We annotate the expression forest by unique pattern ids, all initialized as `*`, e.g.:
+
+        id | name  | rel*
+        ---|-------|-----------------
+        0  | 'rlz' |    (    $nat    <    (    .0    ._    {    {    {}    _}     0}     )    ,     (     .0     rlz     )     ,     ()     >     )     
+           |       | %0 ( %1 $nat %2 < %3 ( %4 .0 %5 ._ %6 { %7 { %8 {} %9 _} %10 0} %11 ) %12, %13 ( %14 .0 %15 rlz %16 ) %17 , %18 () %19 > %20 ) %21
+
+2. The pattern graph (with 22 nodes) has no edges.
+
+### Pattern graph folding
+
+Given a pattern graph, `(V,E)`, and an expression forest, `F`, we calculate a `folding` of the pattern graph,
+which consists of:
+
+1. equivalence classes `eq` over `V + C`
+    - `%x $t $y` implies `%x ~ %y ~ $t`
+    - `%x ( %y ...)` implies `%x ~ %y`
+    - `(... %x ) %y` implies `%x ~ %y`
+    - `%x () %y` implies `%x ~ %y`
+    - `%x { %y1 ..., %y2 ..., ..., %yn ...}` implies `%x ~ %y1 ~ %y2 ~ ... ~ %yn`
+    - `%x < %y1 ..., %y2 ..., ..., %yn ...>` implies `%x ~ %y1 ~ %y2 ~ ... ~ %yn`
+    - `<... %y1, ... %y2, ..., ... %yn > %x` implies `%x ~ %y1 ~ %y2 ~ ... ~ %yn`
+    - if `%x ~ $t` and `%x .field %y` implies `%y ~ $t.field`
+2. new edges, `E'`
+    - `%x .field %y` implies `%x -[field]-> %y`
+    - `{..., ... %x field, ...} %y` implies `%y -[field]-> %x`
+3. pattern refinements, `R`
+    - `{... field} %x` implies `%x: <*>`
+    - `{... field, ...} %y` implies `%y: {}`
+
+In our example:
+
+`eq: {0,1,nat,2,3,4,11,12,13,14,15,16,17,18,19,20,21}, {5}, {6,7,8}, {9}, {10}`
+
+`E': (4 -[0]-> 5), (5 -[_]-> 6), (10 -[_]-> 9), (11 -[0]-> 10), (14 -[0]-> 15)`
+
+`R: 9:{}, 10:<*>`
+
+By folding the pattern graph with those information we get:
+
+    %n ( %n $nat %n < %n ( %n .0 %n ._ %{} { %{} { %{} {} %{} _} %n 0} %n ) %n, %n ( %n .0 %n rlz %n ) %n , %n () %n > %n ) %n
+
+### Another example
+
+    dec = [(),-1] PLUS;
+    zero? = [(),0] EQ 0;
+    factorial = < zero? 1, [dec factorial, ()] TIMES >;
+    { () x, factorial "x!" }
+
+    name   | rel*
+    -------|-----------------
+    PLUS   | %1 $[int] %2 PLUS %3 $int %4  
+    -1     | %5 {} %6 -1 %7 $int %8
+    EQ     | %9 $[int] %10
+    dec    | %11 [ %12 () %13, %14 -1 %15 ] %16 PLUS %17
+    zero?  | %18 [ %19 () %20, %21 0 %22 ] %23 EQ %24 .0 %25
+
+----
+Other observations:
+
+Product like {.x ..., .y ...} implies the input is a product (not really valid)
+Pattern `p = <{ p x, ... }>` implies, it is a union!, i.e., `p = < p x, ... >`, e.g, in `< .x, () >`.
+
+## To do
+
+0. rewrite the algo to be layered:
+    1. pure algebraic (no vectors)
+    2. with support for homogenous vectors
+    3. native support for bytes, integers (`int64` only), booleans, and strings (`UTF8` only)
+1. incremental pattern graph construction
+2. folding pattern graph nodes corresponding to a code (e.g., parts of
+   the pattern graph whose nodes are all closed), adding codes into code graph and, 
+   finally, re-minimizing it.
+3. support for 'built-in' types via plugins.
+4. do not simplify merge for code derivation sake.

package/code-resolution.mjs

@@ -0,0 +1,51 @@
+function resolveCode(rel, inCode, outCode) {
+  const annotatedRel = {inCode, outCode, rel};
+  return annotatedRel;
+}
+
+// pat:  { kind: [null, 'product', 'union'],
+//         closed: [true, false],
+//         fields: [{f: string, pat: pat}, ...]}
+const emptyPat = { kind: 'union', closed: true, fields: [] };
+const starPat = { kind: null, closed: false, fields: [] };
+function isEmptyPat(pat) {
+  const {kind, closed, fields} = pat;
+  return kind === 'union' && closed && fields.length === 0;
+}
+
+function cap(pat1, pat2) {
+  if (pat1 === pat2) return pat1;
+  const {kind:kind1, closed:closed1, fields:fields1} = pat1;
+  const {kind:kind2, closed:closed2, fields:fields2} = pat2;
+  let kind, fields;
+  if (kind1 === null) {
+    kind = kind2;
+  } else if (kind2 === null) {
+    kind = kind1;
+  } else if (kind1 === kind2) {
+    kind = kind1;
+  } else 
+    return emptyPat;
+  
+  const map1 = new Map(fields1.map(({f, pat}) => [f, pat]));
+  const map2 = new Map(fields2.map(({f, pat}) => [f, pat]));
+  const fieldNames = new Set([...map1.keys(), ...map2.keys()]);
+  fields = [...fieldNames].map(f => {
+    const pat1 = map1.get(f) || starPat;
+    const pat2 = map2.get(f) || starPat;
+    const pat = cap(pat1, pat2);
+    return {f, pat};
+  });
+  if (fields.some(({pat}) => isEmptyPat(pat))) {
+    return emptyPat;
+  }
+  if ((closed1 && fields1.length !== fields.length) || (closed2 && fields2.length !== fields.length)) {
+    return emptyPat;
+  }
+  return {kind, closed: closed1 || closed2, fields};
+}  
+
+
+export default { cap, starPat, emptyPat, isEmptyPat, resolveCode };
+export { cap, starPat, emptyPat, isEmptyPat, resolveCode };
+

package/codes.mjs

@@ -1,3 +1,56 @@
+function isBuiltIn(code) {  
+  return code.match(/^(int|string|bool)$/);
+}
+
+function logCeiling(n, base = 2) {
+  return Math.ceil(Math.log(n) / Math.log(base));
+}
+
+function encodeCodeToString(code, codes) {
+  if (isBuiltIn(code)) return code;
+  var i = 0;
+  const Q = [[code,i]];
+  const D = {[code]: i++};
+  var result=[];
+  while (Q.length > 0) {
+    const [x,c] = Q.shift();
+    const c_code = codes[x];
+    if (c_code.code == "vector") {
+      const u_arg = c_code.vector;
+      if (isBuiltIn(u_arg)) {
+        result.push(`$C${result.length}=[${u_arg}];`);
+        continue;
+      }
+      if (D[u_arg] === undefined) {
+        Q.push([u_arg,i]);
+        D[u_arg] = i++;
+      }
+      result.push(`$C${result.length}=[C${D[u_arg]}];`);
+      continue;
+    }
+    const u_args =  Object.keys(c_code[c_code.code]).sort().map((k) => {
+      const u_arg = c_code[c_code.code][k];
+      if (isBuiltIn(u_arg)) 
+        return `${u_arg}${JSON.stringify(k)}`;
+      if (D[u_arg] === undefined) {
+        Q.push([u_arg,i]);
+        D[u_arg] = i++;
+      }
+      return `C${D[u_arg]}${JSON.stringify(k)}`;
+    });
+    if (c_code.code == "union") {
+      //result += `$${c}=<${u_args.join(",")}>;`;
+      result.push(`$C${result.length}=<${u_args.join(",")}>;`);
+    } else if (c_code.code == "product") {
+      //result += `$${c}={${u_args.join(",")}};`;
+      result.push(`$C${result.length}={${u_args.join(",")}};`);
+    } else {
+      throw new Error(`Unexpected code ${c_code.code}`);
+    }
+  }
+  return result.join("");
+};
+
 function are_different(classes, representatives, name1, name2, codes) {
   if (name1 === name2) return false;
   const code1 = codes[name1];
@@ -63,6 +116,7 @@ function minimize(codes) {
       }
     }
   }
+
   return { classes, representatives };
 }
 
@@ -81,7 +135,7 @@ function normalizeAll(codes, representatives) {
       switch (code.code) {
         case "union":
         case "product":
-          normalized[name] = { code: code.code };
+          normalized[name] = { ...code };
           normalized[name][code.code] = normalize(
             code[code.code],
             representatives
@@ -89,7 +143,7 @@ function normalizeAll(codes, representatives) {
           break;
         case "vector":
           normalized[name] = {
-            code: "vector",
+            ...code,
             vector: representatives[code.vector] || code.vector,
           };
       }
@@ -99,5 +153,6 @@ function normalizeAll(codes, representatives) {
   }, {});
 }
 
-export default { minimize, normalize, normalizeAll };
-export { minimize, normalize, normalizeAll };
+
+export default { minimize, normalize, normalizeAll, encodeCodeToString };
+export { minimize, normalize, normalizeAll, encodeCodeToString };

package/filters.mjs

@@ -0,0 +1,59 @@
+// filter is of type: [name|code|null|union|product|vector],
+// null|union|product have fields, and can be open
+
+function unify_with_code(codes, code, filter, context){
+  // console.log({code, filter, context});
+  if (context == undefined) return undefined;
+  if (code == undefined) return undefined;
+  switch (filter.type) {
+    case "code": {
+      if (filter.code == code)
+        return context;
+      return undefined;
+    }
+    case "name": {
+      const code_filter = {type: 'code', code: code};
+      if (filter.name in context) {
+        const new_filter = context[filter.name];
+        const new_context = {... context, [filter.name]: code_filter};
+        return unify_with_code(codes, code, new_filter, new_context);
+      }
+      return {...context, [filter.name]: {type: 'code', code: code}} 
+    }
+    case "product": 
+    case "union":
+    case null: {
+      const ctype = codes[code]?.code;
+      if (ctype != 'product' && ctype != 'union') return undefined;
+      const code_fields = codes[code][ctype];
+      const new_context = Object.keys(filter.fields).reduce((new_context, field) => 
+        unify_with_code(codes, code_fields[field], filter.fields[field], new_context)
+      , {...context});
+      if (filter.open !== true && Object.keys(code_fields).length != Object.keys(filter.fields).length)
+        return undefined;
+      return new_context;
+    }
+    case "vector": {
+      const ctype = codes[code]?.code;
+      if (ctype != 'vector') return undefined;
+      const code_vector = codes[code].vector;
+      return unify_with_code(codes, code_vector, filter.vector, context);
+    }
+  }
+}
+
+// we assume the names occurring in both filters are disjoint
+function unify(codes, filter1, filter2, context ) {
+  if (context == undefined) return undefined;
+  if (filter1 == undefined  ||  filter2 == undefined) return undefined;
+
+  switch (filter1.type) {
+    case "code":
+      return unify_with_code(codes, filter1.code, filter2, context);
+  }
+}
+
+export default { unify_with_code };
+export { unify_with_code };
+
+

package/hash.mjs

@@ -0,0 +1,39 @@
+const baseToShifted = {
+    '0': 'A', '1': 'B', '2': 'C', '3': 'D',
+    '4': 'E', '5': 'F', '6': 'G', '7': 'H',
+    '8': 'I', '9': 'J', 'a': 'K', 'b': 'L',
+    'c': 'M', 'd': 'N', 'e': 'O', 'f': 'P',
+    'g': 'Q', 'h': 'R', 'i': 'S', 'j': 'T',
+    'k': 'U', 'l': 'V', 'm': 'W', 'n': 'X',
+    'o': 'Y', 'p': 'Z', 'q': 'q', 'r': 'r',
+    's': 's', 't': 't', 'u': 'u', 'v': 'v',
+    'w': 'w', 'x': 'x', 'y': 'y', 'z': 'z'
+};
+
+const base = Object.keys(baseToShifted).length;
+
+function shifString(baseString) {
+    let shiftedString = '';
+    for (let i = 0; i < baseString.length; i++) {
+        shiftedString += baseToShifted[baseString[i]];
+    }
+    return shiftedString;
+}
+
+function hash(inputString) {
+    if (inputString.match(/^\$C0=.*;$/))
+        inputString = inputString.slice(4,-1);
+    let hashValue = 0;
+    const prime = 31;
+    const mod = 1000000000039;
+
+    for (let i = 0; i < inputString.length; i++) {
+        let charCode = inputString.charCodeAt(i);
+        hashValue = (hashValue * prime + charCode) % mod;
+    }
+
+    return shifString(hashValue.toString(base));
+}
+
+export default hash;
+export { hash };

package/index.mjs

@@ -1,24 +1,54 @@
 import { parse } from "./parser.mjs";
+import { patterns } from "./patterns.mjs";
 import run from "./run.mjs";
 import t from "./codes.mjs";
+import hash from "./hash.mjs";
 
 function finalize(codes) {
   const representatives = t.minimize(codes).representatives;
+  const normalizedCodes = t.normalizeAll(codes, representatives);
+  const globalNames = Object.keys(normalizedCodes).reduce((globalNames, name) => {
+    const globalDef = t.encodeCodeToString(name, normalizedCodes);
+    normalizedCodes[name].def = globalDef;
+    globalNames[name] = hash(globalDef);
+    return globalNames;
+  }, {});
+  const globalCodes = Object.keys(normalizedCodes).reduce((globalCodes, name) => {
+    globalCodes[globalNames[name]] = normalizedCodes[name];
+    return globalCodes;
+  },{});
+  // console.log("globalCodes",globalCodes);
+
+  const extendedRepresentatives = Object.keys(representatives).reduce((result, name) => {
+    result[name] = globalNames[representatives[name]] || name;
+    return result;
+  }, Object.values(globalNames).reduce((result, name) => ({[name]: name, ...result}),{}));
+  // console.log("extendedRepresentatives",extendedRepresentatives);
+
+  const normalizedGlobalCodes = t.normalizeAll(globalCodes, extendedRepresentatives);
+  // console.log("normalizedGlobalCodes",normalizedGlobalCodes);
+
+  
   return {
-    codes: t.normalizeAll(codes, representatives),
-    representatives,
+    codes: normalizedGlobalCodes,
+    representatives: extendedRepresentatives
   };
 }
 
 function compile(script) {
-  const { defs, exp } = parse(script);
-  const { codes, representatives } = finalize(defs.codes);
-  run.defs = {
-    codes,
-    representatives,
-    rels: defs.rels,
-  };
-  return run.bind(null, exp);
+  try {
+    run.defs = annotate(script);
+  } catch (e) {
+    console.error(e);
+    const { defs, exp } = parse(script);
+    const { codes, representatives } = finalize(defs.codes);
+    run.defs = {
+      rels: {...defs.rels, "__main__": [exp]}, 
+      codes, representatives
+    };
+    console.error("WARN: Recompiled without type reconciliation due to the type error above.");
+  }
+  return run.bind(null, run.defs.rels.__main__[0]);
 }
 
 compile.doc = "Transforms k-script (string) into a function";
@@ -28,5 +58,18 @@ function runScriptOnData(script, data) {
 }
 runScriptOnData.doc = "Run 'script' (string) on 'data': (script,data) -> data";
 
-export default { compile, run: runScriptOnData, parse };
-export { compile, runScriptOnData as run, parse };
+
+function annotate(script) {
+  const { defs, exp } = parse(script);
+  const { codes, representatives } = finalize(defs.codes);
+
+  const rels = {...defs.rels, "__main__": [exp]};
+
+  const pats = patterns(codes, representatives, rels);
+ 
+  return {rels,codes,representatives, ...pats}
+}
+annotate.doc = "Annotate all the script expressions with patterns";
+
+export default { compile, run: runScriptOnData, parse, annotate };
+export { compile, runScriptOnData as run, parse, annotate };

package/package.json

@@ -1,11 +1,11 @@
 {
     "name": "@fraczak/k",
-    "version": "2.1.6",
+    "version": "3.0.2",
     "description": "k-language for JSON-like data transformation",
     "main": "index.mjs",
     "scripts": {
         "prepare": "jiwson -m es6 parser.jison -o parser.mjs",
-        "test": "node test.mjs"
+        "test": "node test.mjs; for f in Code-derivation-tests/*.mjs; do echo $f; node $f; done; node test-fingerprint.mjs"
     },
     "repository": {
         "type": "git",