iscc-core-ts 0.0.5 → 0.0.6

package/README.md

@@ -1,2 +1,44 @@
 # iscc-core-ts
 Iscc core TypeScript implementation of iscc-core reference
+
+
+
+# Prerequiste
+
+We recomand to use [nvm](https://github.com/nvm-sh/nvm) to install and target node and npm versions. [Here](https://www.freecodecamp.org/news/node-version-manager-nvm-install-guide/) an NVM install guide.
+
+Nvm version used:
+```sh
+nvm --version
+0.39.3
+```
+Install node 21 
+
+```sh
+nvm install 21
+v21.7.1 is already installed.
+Now using node v21.7.1 (npm v10.5.0)
+```
+
+# Install and build
+
+```sh
+npm i
+npm run build
+```
+
+# Tests
+
+```sh
+npm run test
+npm run test-isolated
+npm run test-esm
+npm run test-esm-isolated
+```
+
+
+
+
+
+
+

package/lib/cjs/codec.d.ts

@@ -0,0 +1,7 @@
+import { MainTypes, SubTypes, Version } from './constants';
+export declare function toHexString(bytes: Uint8Array): string;
+export declare function encode_length(mtype: MainTypes, length: number): number;
+export declare function encode_header(mtype: MainTypes, stype: SubTypes, version: Version, length: number): string;
+export declare function encode_varnibble(n: number): string;
+export declare function encode_component(mtype: MainTypes, stype: SubTypes, version: Version, bit_length: number, digest: string): string;
+export declare function encode_base32(data: string): string;

package/lib/cjs/codec.js

@@ -0,0 +1,121 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.encode_base32 = exports.encode_component = exports.encode_varnibble = exports.encode_header = exports.encode_length = exports.toHexString = void 0;
+const rfc4648_1 = require("rfc4648");
+const utils_1 = require("./utils");
+const constants_1 = require("./constants");
+function toHexString(bytes) {
+    let result = '';
+    for (const byte of bytes) {
+        if (byte > 15) {
+            result += (byte & 0xff).toString(16);
+        }
+        else {
+            result += '0' + (byte & 0xff).toString(16);
+        }
+    }
+    return result;
+}
+exports.toHexString = toHexString;
+function encode_length(mtype, length) {
+    const error = 'Invalid length [' + length + '] for MainType [' + mtype + ']';
+    if (mtype == constants_1.MainTypes.META ||
+        mtype == constants_1.MainTypes.SEMANTIC ||
+        mtype == constants_1.MainTypes.CONTENT ||
+        mtype == constants_1.MainTypes.DATA ||
+        mtype == constants_1.MainTypes.INSTANCE ||
+        mtype == constants_1.MainTypes.FLAKE) {
+        if (length >= 32 && length % 32 == 0) {
+            return Math.floor(length / 32) - 1;
+        }
+        else {
+            throw Error(error);
+        }
+    }
+    else if (mtype == constants_1.MainTypes.ISCC) {
+        if (0 <= length && length <= 7) {
+            return length;
+        }
+        else {
+            throw Error(error);
+        }
+    }
+    else if (mtype == constants_1.MainTypes.ID) {
+        if (64 <= length && length <= 96) {
+            return Math.floor((length - 64) / 8);
+        }
+        else {
+            throw Error('MainType [' + mtype + '] is not a unit');
+        }
+    }
+    else {
+        throw Error(error);
+    }
+}
+exports.encode_length = encode_length;
+function encode_header(mtype, stype, version, length) {
+    let header = '';
+    for (const n of [
+        mtype.valueOf(),
+        stype.valueOf(),
+        version.valueOf(),
+        length
+    ]) {
+        header += encode_varnibble(n);
+    }
+    return toHexString((0, utils_1.binaryArrayToUint8Array)(header));
+}
+exports.encode_header = encode_header;
+function encode_varnibble(n) {
+    if (0 <= n && n < 8) {
+        return n.toString(2).padStart(4, '0');
+        //  return new BitSet(n).toString().padStart(4,"0");
+    }
+    else if (8 <= n && n < 72) {
+        //  return new BitSet("10").toString() + new BitSet(n-8).toString().padStart(6,"0");
+        return '10' + (n - 8).toString(2).padStart(6, '0');
+    }
+    else if (72 <= n && n < 584) {
+        return '110' + (n - 72).toString(2).padStart(9, '0');
+        //  return new BitSet("110").toString() + new BitSet(n-72).toString().padStart(9,"0");
+    }
+    else if (584 <= n && n < 4680) {
+        return '1110' + (n - 584).toString(2).padStart(12, '0');
+        // return new BitSet("1110").toString() + new BitSet(n-584).toString().padStart(12,"0");
+    }
+    else {
+        throw Error('encode_varnibble - Value must be between 0 and 4679');
+    }
+}
+exports.encode_varnibble = encode_varnibble;
+function encode_component(mtype, stype, version, bit_length, digest) {
+    if (mtype == constants_1.MainTypes.META ||
+        mtype == constants_1.MainTypes.SEMANTIC ||
+        mtype == constants_1.MainTypes.CONTENT ||
+        mtype == constants_1.MainTypes.DATA ||
+        mtype == constants_1.MainTypes.INSTANCE ||
+        mtype == constants_1.MainTypes.ID ||
+        mtype == constants_1.MainTypes.FLAKE) {
+        const encoded_length = encode_length(mtype, bit_length);
+        const nbytes = Math.floor(bit_length / 8);
+        const header = encode_header(mtype, stype, version, encoded_length);
+        const body = digest.substring(0, nbytes * 2);
+        const component_code = encode_base32(header + body);
+        return component_code;
+    }
+    else if (mtype == constants_1.MainTypes.ISCC) {
+        throw Error('{mtype} ISCC is not a unit');
+    }
+    else {
+        throw Error('Illegal MainType');
+    }
+}
+exports.encode_component = encode_component;
+function encode_base32(data) {
+    /*
+    Standard RFC4648 base32 encoding without padding.
+  */
+    return (0, utils_1.rtrim)(rfc4648_1.base32.stringify(Buffer.from(data, 'hex')), '=');
+}
+exports.encode_base32 = encode_base32;
+//# sourceMappingURL=codec.js.map

package/lib/cjs/codec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"codec.js","sourceRoot":"","sources":["../../src/codec.ts"],"names":[],"mappings":";;;AAAA,qCAAiC;AACjC,mCAAyD;AACzD,2CAA2D;AAE3D,SAAgB,WAAW,CAAC,KAAiB;IACzC,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,IAAI,GAAG,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACzC,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC/C,CAAC;IACL,CAAC;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAVD,kCAUC;AAED,SAAgB,aAAa,CAAC,KAAgB,EAAE,MAAc;IAC1D,MAAM,KAAK,GACP,kBAAkB,GAAG,MAAM,GAAG,kBAAkB,GAAG,KAAK,GAAG,GAAG,CAAC;IACnE,IACI,KAAK,IAAI,qBAAS,CAAC,IAAI;QACvB,KAAK,IAAI,qBAAS,CAAC,QAAQ;QAC3B,KAAK,IAAI,qBAAS,CAAC,OAAO;QAC1B,KAAK,IAAI,qBAAS,CAAC,IAAI;QACvB,KAAK,IAAI,qBAAS,CAAC,QAAQ;QAC3B,KAAK,IAAI,qBAAS,CAAC,KAAK,EAC1B,CAAC;QACC,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,GAAG,EAAE,IAAI,CAAC,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;QACvC,CAAC;aAAM,CAAC;YACJ,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;IACL,CAAC;SAAM,IAAI,KAAK,IAAI,qBAAS,CAAC,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC,IAAI,MAAM,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;YAC7B,OAAO,MAAM,CAAC;QAClB,CAAC;aAAM,CAAC;YACJ,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;IACL,CAAC;SAAM,IAAI,KAAK,IAAI,qBAAS,CAAC,EAAE,EAAE,CAAC;QAC/B,IAAI,EAAE,IAAI,MAAM,IAAI,MAAM,IAAI,EAAE,EAAE,CAAC;YAC/B,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACzC,CAAC;aAAM,CAAC;YACJ,MAAM,KAAK,CAAC,YAAY,GAAG,KAAK,GAAG,iBAAiB,CAAC,CAAC;QAC1D,CAAC;IACL,CAAC;SAAM,CAAC;QACJ,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;AACL,CAAC;AA/BD,sCA+BC;AAED,SAAgB,aAAa,CACzB,KAAgB,EAChB,KAAe,EACf,OAAgB,EAChB,MAAc;IAEd,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI;QACZ,KAAK,CAAC,OAAO,EAAE;QACf,KAAK,CAAC,OAAO,EAAE;QACf,OAAO,CAAC,OAAO,EAAE;QACjB,MAAM;KACT,EAAE,CAAC;QACA,MAAM,IAAI,gBAAgB,CAAC,CAAC,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,WAAW,CAAC,IAAA,+BAAuB,EAAC,MAAM,CAAC,CAAC,CAAC;AACxD,CAAC;AAhBD,sCAgBC;AAED,SAAgB,gBAAgB,CAAC,CAAS;IACtC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAClB,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACtC,oDAAoD;IACxD,CAAC;SAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QAC1B,oFAAoF;QACpF,OAAO,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACvD,CAAC;SAAM,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QAC5B,OAAO,KAAK,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACrD,sFAAsF;IAC1F,CAAC;SAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;QAC9B,OAAO,MAAM,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QACxD,wFAAwF;IAC5F,CAAC;SAAM,CAAC;QACJ,MAAM,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACvE,CAAC;AACL,CAAC;AAhBD,4CAgBC;AAED,SAAgB,gBAAgB,CAC5B,KAAgB,EAChB,KAAe,EACf,OAAgB,EAChB,UAAkB,EAClB,MAAc;IAEd,IACI,KAAK,IAAI,qBAAS,CAAC,IAAI;QACvB,KAAK,IAAI,qBAAS,CAAC,QAAQ;QAC3B,KAAK,IAAI,qBAAS,CAAC,OAAO;QAC1B,KAAK,IAAI,qBAAS,CAAC,IAAI;QACvB,KAAK,IAAI,qBAAS,CAAC,QAAQ;QAC3B,KAAK,IAAI,qBAAS,CAAC,EAAE;QACrB,KAAK,IAAI,qBAAS,CAAC,KAAK,EAC1B,CAAC;QACC,MAAM,cAAc,GAAW,aAAa,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAChE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;QAC1C,MAAM,MAAM,GAAG,aAAa,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC;QACpE,MAAM,IAAI,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;QAC7C,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;QACpD,OAAO,cAAc,CAAC;IAC1B,CAAC;SAAM,IAAI,KAAK,IAAI,qBAAS,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAC9C,CAAC;SAAM,CAAC;QACJ,MAAM,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACpC,CAAC;AACL,CAAC;AA3BD,4CA2BC;AAED,SAAgB,aAAa,CAAC,IAAY;IACtC;;IAEA;IACA,OAAO,IAAA,aAAK,EAAC,gBAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAClE,CAAC;AALD,sCAKC","sourcesContent":["import { base32 } from 'rfc4648';\nimport { binaryArrayToUint8Array, rtrim } from './utils';\nimport { MainTypes, SubTypes, Version } from './constants';\n\nexport function toHexString(bytes: Uint8Array): string {\n    let result = '';\n    for (const byte of bytes) {\n        if (byte > 15) {\n            result += (byte & 0xff).toString(16);\n        } else {\n            result += '0' + (byte & 0xff).toString(16);\n        }\n    }\n    return result;\n}\n\nexport function encode_length(mtype: MainTypes, length: number): number {\n    const error =\n        'Invalid length [' + length + '] for MainType [' + mtype + ']';\n    if (\n        mtype == MainTypes.META ||\n        mtype == MainTypes.SEMANTIC ||\n        mtype == MainTypes.CONTENT ||\n        mtype == MainTypes.DATA ||\n        mtype == MainTypes.INSTANCE ||\n        mtype == MainTypes.FLAKE\n    ) {\n        if (length >= 32 && length % 32 == 0) {\n            return Math.floor(length / 32) - 1;\n        } else {\n            throw Error(error);\n        }\n    } else if (mtype == MainTypes.ISCC) {\n        if (0 <= length && length <= 7) {\n            return length;\n        } else {\n            throw Error(error);\n        }\n    } else if (mtype == MainTypes.ID) {\n        if (64 <= length && length <= 96) {\n            return Math.floor((length - 64) / 8);\n        } else {\n            throw Error('MainType [' + mtype + '] is not a unit');\n        }\n    } else {\n        throw Error(error);\n    }\n}\n\nexport function encode_header(\n    mtype: MainTypes,\n    stype: SubTypes,\n    version: Version,\n    length: number\n): string {\n    let header = '';\n    for (const n of [\n        mtype.valueOf(),\n        stype.valueOf(),\n        version.valueOf(),\n        length\n    ]) {\n        header += encode_varnibble(n);\n    }\n    return toHexString(binaryArrayToUint8Array(header));\n}\n\nexport function encode_varnibble(n: number): string {\n    if (0 <= n && n < 8) {\n        return n.toString(2).padStart(4, '0');\n        //  return new BitSet(n).toString().padStart(4,\"0\");\n    } else if (8 <= n && n < 72) {\n        //  return new BitSet(\"10\").toString() + new BitSet(n-8).toString().padStart(6,\"0\");\n        return '10' + (n - 8).toString(2).padStart(6, '0');\n    } else if (72 <= n && n < 584) {\n        return '110' + (n - 72).toString(2).padStart(9, '0');\n        //  return new BitSet(\"110\").toString() + new BitSet(n-72).toString().padStart(9,\"0\");\n    } else if (584 <= n && n < 4680) {\n        return '1110' + (n - 584).toString(2).padStart(12, '0');\n        // return new BitSet(\"1110\").toString() + new BitSet(n-584).toString().padStart(12,\"0\");\n    } else {\n        throw Error('encode_varnibble - Value must be between 0 and 4679');\n    }\n}\n\nexport function encode_component(\n    mtype: MainTypes,\n    stype: SubTypes,\n    version: Version,\n    bit_length: number,\n    digest: string\n): string {\n    if (\n        mtype == MainTypes.META ||\n        mtype == MainTypes.SEMANTIC ||\n        mtype == MainTypes.CONTENT ||\n        mtype == MainTypes.DATA ||\n        mtype == MainTypes.INSTANCE ||\n        mtype == MainTypes.ID ||\n        mtype == MainTypes.FLAKE\n    ) {\n        const encoded_length: number = encode_length(mtype, bit_length);\n        const nbytes = Math.floor(bit_length / 8);\n        const header = encode_header(mtype, stype, version, encoded_length);\n        const body = digest.substring(0, nbytes * 2);\n        const component_code = encode_base32(header + body);\n        return component_code;\n    } else if (mtype == MainTypes.ISCC) {\n        throw Error('{mtype} ISCC is not a unit');\n    } else {\n        throw Error('Illegal MainType');\n    }\n}\n\nexport function encode_base32(data: string) {\n    /*\n    Standard RFC4648 base32 encoding without padding.\n  */\n    return rtrim(base32.stringify(Buffer.from(data, 'hex')), '=');\n}\n"]}

package/lib/cjs/constants.d.ts

@@ -0,0 +1,44 @@
+export declare const METACODE_BITS: number;
+export declare const META_TRIM_NAME: number;
+export declare const META_NGRAM_SIZE_TEXT: number;
+export declare const META_NGRAM_SIZE_BYTES: number;
+/**
+ *
+ *     """
+    ## MT - MainTypes
+
+    | Uint | Symbol   | Bits | Purpose                                                 |
+    |----- |:---------|------|---------------------------------------------------------|
+    | 0    | META     | 0000 | Match on metadata similarity                            |
+    | 1    | SEMANTIC | 0001 | Match on semantic content similarity                    |
+    | 2    | CONTENT  | 0010 | Match on perceptual content similarity                  |
+    | 3    | DATA     | 0011 | Match on data similarity                                |
+    | 4    | INSTANCE | 0100 | Match on data identity                                  |
+    | 5    | ISCC     | 0101 | Composite of two or more ISCC-UNITs with common header  |
+    """
+
+ */
+export declare enum MainTypes {
+    META = 0,
+    SEMANTIC = 1,
+    CONTENT = 2,
+    DATA = 3,
+    INSTANCE = 4,
+    ISCC = 5,
+    ID = 6,
+    FLAKE = 7
+}
+export declare enum SubTypes {
+    NONE = 0
+}
+export declare enum Version {
+    V0 = 0
+}
+export interface IMetaCodeResult {
+    iscc: string;
+    metahash: string;
+    name: string;
+    description?: string;
+    meta?: string;
+    version: number;
+}

package/lib/cjs/constants.js

@@ -0,0 +1,67 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Version = exports.SubTypes = exports.MainTypes = exports.META_NGRAM_SIZE_BYTES = exports.META_NGRAM_SIZE_TEXT = exports.META_TRIM_NAME = exports.METACODE_BITS = void 0;
+exports.METACODE_BITS = 64;
+exports.META_TRIM_NAME = 128;
+exports.META_NGRAM_SIZE_TEXT = 3;
+exports.META_NGRAM_SIZE_BYTES = 4;
+/**
+ *
+ *     """
+    ## MT - MainTypes
+
+    | Uint | Symbol   | Bits | Purpose                                                 |
+    |----- |:---------|------|---------------------------------------------------------|
+    | 0    | META     | 0000 | Match on metadata similarity                            |
+    | 1    | SEMANTIC | 0001 | Match on semantic content similarity                    |
+    | 2    | CONTENT  | 0010 | Match on perceptual content similarity                  |
+    | 3    | DATA     | 0011 | Match on data similarity                                |
+    | 4    | INSTANCE | 0100 | Match on data identity                                  |
+    | 5    | ISCC     | 0101 | Composite of two or more ISCC-UNITs with common header  |
+    """
+
+ */
+var MainTypes;
+(function (MainTypes) {
+    MainTypes[MainTypes["META"] = 0] = "META";
+    MainTypes[MainTypes["SEMANTIC"] = 1] = "SEMANTIC";
+    MainTypes[MainTypes["CONTENT"] = 2] = "CONTENT";
+    MainTypes[MainTypes["DATA"] = 3] = "DATA";
+    MainTypes[MainTypes["INSTANCE"] = 4] = "INSTANCE";
+    MainTypes[MainTypes["ISCC"] = 5] = "ISCC";
+    MainTypes[MainTypes["ID"] = 6] = "ID";
+    MainTypes[MainTypes["FLAKE"] = 7] = "FLAKE";
+})(MainTypes || (exports.MainTypes = MainTypes = {}));
+/*
+      ## ST - SubTypes
+    
+      | Uint | Symbol   | Bits | Purpose                                                 |
+      |----- |:---------|------|---------------------------------------------------------|
+      | 0    | NONE     | 0000 | For MainTypes that do not specify SubTypes              |
+      """
+    
+      NONE = 0
+    */
+var SubTypes;
+(function (SubTypes) {
+    SubTypes[SubTypes["NONE"] = 0] = "NONE";
+})(SubTypes || (exports.SubTypes = SubTypes = {}));
+/*
+        ## VS - Version
+    
+        Code Version
+    
+        | Uint | Symbol   | Bits | Purpose                                                 |
+        |----- |:---------|------|---------------------------------------------------------|
+        | 0    | V0       | 0000 | Initial Version of Code without breaking changes        |
+    
+        """
+    
+        V0 = 0
+    
+    */
+var Version;
+(function (Version) {
+    Version[Version["V0"] = 0] = "V0";
+})(Version || (exports.Version = Version = {}));
+//# sourceMappingURL=constants.js.map

package/lib/cjs/constants.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../src/constants.ts"],"names":[],"mappings":";;;AAAa,QAAA,aAAa,GAAW,EAAE,CAAC;AAC3B,QAAA,cAAc,GAAW,GAAG,CAAC;AAC7B,QAAA,oBAAoB,GAAW,CAAC,CAAC;AACjC,QAAA,qBAAqB,GAAW,CAAC,CAAC;AAE/C;;;;;;;;;;;;;;;GAeG;AACH,IAAY,SASX;AATD,WAAY,SAAS;IACjB,yCAAQ,CAAA;IACR,iDAAY,CAAA;IACZ,+CAAW,CAAA;IACX,yCAAQ,CAAA;IACR,iDAAY,CAAA;IACZ,yCAAQ,CAAA;IACR,qCAAM,CAAA;IACN,2CAAS,CAAA;AACb,CAAC,EATW,SAAS,yBAAT,SAAS,QASpB;AAED;;;;;;;;;MASM;AAEN,IAAY,QAEX;AAFD,WAAY,QAAQ;IAChB,uCAAQ,CAAA;AACZ,CAAC,EAFW,QAAQ,wBAAR,QAAQ,QAEnB;AAED;;;;;;;;;;;;;MAaM;AAEN,IAAY,OAEX;AAFD,WAAY,OAAO;IACf,iCAAM,CAAA;AACV,CAAC,EAFW,OAAO,uBAAP,OAAO,QAElB","sourcesContent":["export const METACODE_BITS: number = 64;\nexport const META_TRIM_NAME: number = 128;\nexport const META_NGRAM_SIZE_TEXT: number = 3;\nexport const META_NGRAM_SIZE_BYTES: number = 4;\n\n/**\n * \n *     \"\"\"\n    ## MT - MainTypes\n\n    | Uint | Symbol   | Bits | Purpose                                                 |\n    |----- |:---------|------|---------------------------------------------------------|\n    | 0    | META     | 0000 | Match on metadata similarity                            |\n    | 1    | SEMANTIC | 0001 | Match on semantic content similarity                    |\n    | 2    | CONTENT  | 0010 | Match on perceptual content similarity                  |\n    | 3    | DATA     | 0011 | Match on data similarity                                |\n    | 4    | INSTANCE | 0100 | Match on data identity                                  |\n    | 5    | ISCC     | 0101 | Composite of two or more ISCC-UNITs with common header  |\n    \"\"\"\n\n */\nexport enum MainTypes {\n    META = 0,\n    SEMANTIC = 1,\n    CONTENT = 2,\n    DATA = 3,\n    INSTANCE = 4,\n    ISCC = 5,\n    ID = 6,\n    FLAKE = 7\n}\n\n/*\n      ## ST - SubTypes\n    \n      | Uint | Symbol   | Bits | Purpose                                                 |\n      |----- |:---------|------|---------------------------------------------------------|\n      | 0    | NONE     | 0000 | For MainTypes that do not specify SubTypes              |\n      \"\"\"\n    \n      NONE = 0\n    */\n\nexport enum SubTypes {\n    NONE = 0\n}\n\n/*\n        ## VS - Version\n    \n        Code Version\n    \n        | Uint | Symbol   | Bits | Purpose                                                 |\n        |----- |:---------|------|---------------------------------------------------------|\n        | 0    | V0       | 0000 | Initial Version of Code without breaking changes        |\n    \n        \"\"\"\n    \n        V0 = 0\n    \n    */\n\nexport enum Version {\n    V0 = 0\n}\n\nexport interface IMetaCodeResult {\n    iscc: string;\n    metahash: string;\n    name: string;\n    description?: string;\n    meta?: string;\n    version: number;\n}\n"]}

package/lib/cjs/content-normalization.d.ts

@@ -0,0 +1,20 @@
+/**
+ *
+ *
+ * Specification implementaton of https://iscc.codes/specification/#text_trim
+ *
+ * @param text
+ * @returns
+ */
+export declare function text_trim(text: string, limit?: number): string;
+export declare function text_encodeUTF8(text: string, limit?: number): string;
+export declare function text_remove_newlines(text: string): string;
+/**
+ *
+ * For category name unicode filtering, the reference is https://unicode.org/Public/UCD/latest/ucd/PropertyValueAliases.txt
+ *
+ * @param text
+ * @returns text normalized
+ */
+export declare function text_clean(text: string): string;
+export declare function text_collapse(text: string): string;

package/lib/cjs/content-normalization.js

@@ -0,0 +1,151 @@
+"use strict";
+/**
+ *
+ *
+ * Specification implementaton of https://iscc.codes/specification/#text_trim
+ *
+ * @param text
+ * @returns
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.text_collapse = exports.text_clean = exports.text_remove_newlines = exports.text_encodeUTF8 = exports.text_trim = void 0;
+const xregexp_1 = __importDefault(require("xregexp"));
+function text_trim(text, limit) {
+    return text_encodeUTF8(text, limit).trim();
+}
+exports.text_trim = text_trim;
+function text_encodeUTF8(text, limit) {
+    const encoder = new TextEncoder();
+    const utf8Arr = encoder.encode(text);
+    const decoder = new TextDecoder('utf-8');
+    if (limit && limit > 0) {
+        text = decoder.decode(utf8Arr.slice(0, limit));
+    }
+    else {
+        text = decoder.decode(utf8Arr);
+    }
+    return text;
+}
+exports.text_encodeUTF8 = text_encodeUTF8;
+function text_remove_newlines(text) {
+    return text
+        .split(/(\s+)/)
+        .filter((x) => x.trim().length > 0)
+        .join(' ');
+}
+exports.text_remove_newlines = text_remove_newlines;
+function isCharControlCategoryUnicode(s) {
+    const cc = (0, xregexp_1.default)('^\\p{Control}+$');
+    const cf = (0, xregexp_1.default)('^\\p{Format}+$');
+    const cs = (0, xregexp_1.default)('^\\p{Surrogate}+$');
+    const co = (0, xregexp_1.default)('^\\p{Private_Use}+$');
+    const cn = (0, xregexp_1.default)('^\\p{Unassigned}+$');
+    if (cc.test(s) || cf.test(s) || cs.test(s) || co.test(s) || cn.test(s)) {
+        return true;
+    }
+    else {
+        return false;
+    }
+}
+function isCharNewLinesUnicode(s) {
+    const regex = /\u000A|\u000B|\u000C|\u000D|\u0085|\u2028|\u2029/;
+    if (regex.test(s)) {
+        return true;
+    }
+    else {
+        return false;
+    }
+}
+/**
+ *
+ * For category name unicode filtering, the reference is https://unicode.org/Public/UCD/latest/ucd/PropertyValueAliases.txt
+ *
+ * @param text
+ * @returns text normalized
+ */
+function text_clean(text) {
+    text = text.normalize('NFKC');
+    let textWithoutCC = '';
+    const charsWithCC = Array.from(text);
+    // Remove control characters
+    for (const c of charsWithCC) {
+        if (!isCharControlCategoryUnicode(c) || isCharNewLinesUnicode(c)) {
+            textWithoutCC = textWithoutCC.concat(c);
+        }
+    }
+    let textFiltered = '';
+    const chars = Array.from(textWithoutCC);
+    let newline_count = 0;
+    // console.log("text"+text)
+    for (const c of chars) {
+        if (isCharNewLinesUnicode(c)) {
+            //console.log("new line!")
+            if (newline_count < 2) {
+                // console.log("new line <2")
+                textFiltered = textFiltered.concat('\u{000A}');
+                newline_count += 1;
+            }
+            continue;
+        }
+        else {
+            newline_count = 0;
+        }
+        textFiltered = textFiltered.concat(c);
+    }
+    text = textFiltered;
+    /* Keep these control characters (Cc) that are commonly considered white-space:
+     */
+    // \u0009, # Horizontal Tab (TAB)
+    //text = text.replace(/\u0009/gu, ' ');
+    // \u000A, # Linefeed
+    //text = text.replace(/\u000A/gu, ' ');
+    // \u000D, # Carriage Return (CR)
+    //text = text.replace(/\u000D/gu, ' ');
+    // 6. Keep or remove whitespace (remove duplicate whitespace) according to keep_ws
+    //  if(!keep_ws){
+    //      text = text.replace(/\s+/g, ' ');
+    // }
+    return text.trim();
+}
+exports.text_clean = text_clean;
+function text_collapse(text) {
+    text = text.normalize('NFD');
+    text = text_trim(text);
+    text = text.replace(/\s/g, '');
+    text = text.toLowerCase();
+    // Filter out all characters that fall into the Unicode categories listed in the constant UNICODE_FILTER: "Cc", "Cf", "Cn", "Co", "Cs", "Mc", "Me", "Mn", "Pc", "Pd", "Pe", "Pf", "Pi", "Po", "Ps"
+    /*
+    gc ; C                                ; Other                            # Cc | Cf | Cn | Co | Cs
+    gc ; Cc                               ; Control                          ; cntrl
+    gc ; Cf                               ; Format
+    gc ; Cn                               ; Unassigned
+    gc ; Co                               ; Private_Use
+    gc ; Cs                               ; Surrogate
+    */
+    text = text.replace(/\p{Other}/gu, '');
+    /*
+    gc ; M                                ; Mark                             ; Combining_Mark                   # Mc | Me | Mn
+    gc ; Mc                               ; Spacing_Mark
+    gc ; Me                               ; Enclosing_Mark
+    gc ; Mn                               ; Nonspacing_Mark
+    */
+    text = text.replace(/\p{Mark}/gu, '');
+    /*
+    gc ; P                                ; Punctuation                      ; punct                            # Pc | Pd | Pe | Pf | Pi | Po | Ps
+    gc ; Pc                               ; Connector_Punctuation
+    gc ; Pd                               ; Dash_Punctuation
+    gc ; Pe                               ; Close_Punctuation
+    gc ; Pf                               ; Final_Punctuation
+    gc ; Pi                               ; Initial_Punctuation
+    gc ; Po                               ; Other_Punctuation
+    gc ; Ps                               ; Open_Punctuation
+    */
+    text = text.replace(/\p{Punctuation}/gu, '');
+    text = text.normalize('NFKC');
+    return text;
+}
+exports.text_collapse = text_collapse;
+//# sourceMappingURL=content-normalization.js.map

package/lib/cjs/content-normalization.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"content-normalization.js","sourceRoot":"","sources":["../../src/content-normalization.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;;;;;AAEH,sDAA8B;AAE9B,SAAgB,SAAS,CAAC,IAAY,EAAE,KAAc;IAClD,OAAO,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;AAC/C,CAAC;AAFD,8BAEC;AAED,SAAgB,eAAe,CAAC,IAAY,EAAE,KAAc;IACxD,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAClC,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACrC,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACrB,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IACnD,CAAC;SAAM,CAAC;QACJ,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAVD,0CAUC;AAED,SAAgB,oBAAoB,CAAC,IAAY;IAC7C,OAAO,IAAI;SACN,KAAK,CAAC,OAAO,CAAC;SACd,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;SAClC,IAAI,CAAC,GAAG,CAAC,CAAC;AACnB,CAAC;AALD,oDAKC;AAED,SAAS,4BAA4B,CAAC,CAAS;IAC3C,MAAM,EAAE,GAAG,IAAA,iBAAO,EAAC,iBAAiB,CAAC,CAAC;IACtC,MAAM,EAAE,GAAG,IAAA,iBAAO,EAAC,gBAAgB,CAAC,CAAC;IACrC,MAAM,EAAE,GAAG,IAAA,iBAAO,EAAC,mBAAmB,CAAC,CAAC;IACxC,MAAM,EAAE,GAAG,IAAA,iBAAO,EAAC,qBAAqB,CAAC,CAAC;IAC1C,MAAM,EAAE,GAAG,IAAA,iBAAO,EAAC,oBAAoB,CAAC,CAAC;IACzC,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QACrE,OAAO,IAAI,CAAC;IAChB,CAAC;SAAM,CAAC;QACJ,OAAO,KAAK,CAAC;IACjB,CAAC;AACL,CAAC;AAED,SAAS,qBAAqB,CAAC,CAAS;IACpC,MAAM,KAAK,GAAG,kDAAkD,CAAC;IACjE,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IAChB,CAAC;SAAM,CAAC;QACJ,OAAO,KAAK,CAAC;IACjB,CAAC;AACL,CAAC;AACD;;;;;;GAMG;AAEH,SAAgB,UAAU,CAAC,IAAY;IACnC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAE9B,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrC,4BAA4B;IAC5B,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC1B,IAAI,CAAC,4BAA4B,CAAC,CAAC,CAAC,IAAI,qBAAqB,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/D,aAAa,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC5C,CAAC;IACL,CAAC;IAED,IAAI,YAAY,GAAG,EAAE,CAAC;IACtB,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACxC,IAAI,aAAa,GAAW,CAAC,CAAC;IAC9B,2BAA2B;IAC3B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACpB,IAAI,qBAAqB,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3B,0BAA0B;YAC1B,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;gBACpB,6BAA6B;gBAC7B,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;gBAC/C,aAAa,IAAI,CAAC,CAAC;YACvB,CAAC;YACD,SAAS;QACb,CAAC;aAAM,CAAC;YACJ,aAAa,GAAG,CAAC,CAAC;QACtB,CAAC;QACD,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,IAAI,GAAG,YAAY,CAAC;IAEpB;OACG;IAEH,iCAAiC;IACjC,uCAAuC;IAEvC,qBAAqB;IACrB,uCAAuC;IAEvC,iCAAiC;IACjC,uCAAuC;IAEvC,kFAAkF;IAClF,iBAAiB;IACjB,yCAAyC;IACzC,IAAI;IAEJ,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;AACvB,CAAC;AAlDD,gCAkDC;AAED,SAAgB,aAAa,CAAC,IAAY;IACtC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACvB,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC/B,IAAI,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IAE1B,kMAAkM;IAClM;;;;;;;MAOE;IACF,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;IAEvC;;;;;MAKE;IACF,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IAEtC;;;;;;;;;MASE;IACF,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC;IAE7C,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAE9B,OAAO,IAAI,CAAC;AAChB,CAAC;AAxCD,sCAwCC","sourcesContent":["/**\n *\n *\n * Specification implementaton of https://iscc.codes/specification/#text_trim\n *\n * @param text\n * @returns\n */\n\nimport XRegExp from 'xregexp';\n\nexport function text_trim(text: string, limit?: number): string {\n    return text_encodeUTF8(text, limit).trim();\n}\n\nexport function text_encodeUTF8(text: string, limit?: number): string {\n    const encoder = new TextEncoder();\n    const utf8Arr = encoder.encode(text);\n    const decoder = new TextDecoder('utf-8');\n    if (limit && limit > 0) {\n        text = decoder.decode(utf8Arr.slice(0, limit));\n    } else {\n        text = decoder.decode(utf8Arr);\n    }\n    return text;\n}\n\nexport function text_remove_newlines(text: string): string {\n    return text\n        .split(/(\\s+)/)\n        .filter((x) => x.trim().length > 0)\n        .join(' ');\n}\n\nfunction isCharControlCategoryUnicode(s: string) {\n    const cc = XRegExp('^\\\\p{Control}+$');\n    const cf = XRegExp('^\\\\p{Format}+$');\n    const cs = XRegExp('^\\\\p{Surrogate}+$');\n    const co = XRegExp('^\\\\p{Private_Use}+$');\n    const cn = XRegExp('^\\\\p{Unassigned}+$');\n    if (cc.test(s) || cf.test(s) || cs.test(s) || co.test(s) || cn.test(s)) {\n        return true;\n    } else {\n        return false;\n    }\n}\n\nfunction isCharNewLinesUnicode(s: string) {\n    const regex = /\\u000A|\\u000B|\\u000C|\\u000D|\\u0085|\\u2028|\\u2029/;\n    if (regex.test(s)) {\n        return true;\n    } else {\n        return false;\n    }\n}\n/**\n *\n * For category name unicode filtering, the reference is https://unicode.org/Public/UCD/latest/ucd/PropertyValueAliases.txt\n *\n * @param text\n * @returns text normalized\n */\n\nexport function text_clean(text: string): string {\n    text = text.normalize('NFKC');\n\n    let textWithoutCC = '';\n    const charsWithCC = Array.from(text);\n    // Remove control characters\n    for (const c of charsWithCC) {\n        if (!isCharControlCategoryUnicode(c) || isCharNewLinesUnicode(c)) {\n            textWithoutCC = textWithoutCC.concat(c);\n        }\n    }\n\n    let textFiltered = '';\n    const chars = Array.from(textWithoutCC);\n    let newline_count: number = 0;\n    // console.log(\"text\"+text)\n    for (const c of chars) {\n        if (isCharNewLinesUnicode(c)) {\n            //console.log(\"new line!\")\n            if (newline_count < 2) {\n                // console.log(\"new line <2\")\n                textFiltered = textFiltered.concat('\\u{000A}');\n                newline_count += 1;\n            }\n            continue;\n        } else {\n            newline_count = 0;\n        }\n        textFiltered = textFiltered.concat(c);\n    }\n    text = textFiltered;\n\n    /* Keep these control characters (Cc) that are commonly considered white-space:\n     */\n\n    // \\u0009, # Horizontal Tab (TAB)\n    //text = text.replace(/\\u0009/gu, ' ');\n\n    // \\u000A, # Linefeed\n    //text = text.replace(/\\u000A/gu, ' ');\n\n    // \\u000D, # Carriage Return (CR)\n    //text = text.replace(/\\u000D/gu, ' ');\n\n    // 6. Keep or remove whitespace (remove duplicate whitespace) according to keep_ws\n    //  if(!keep_ws){\n    //      text = text.replace(/\\s+/g, ' ');\n    // }\n\n    return text.trim();\n}\n\nexport function text_collapse(text: string): string {\n    text = text.normalize('NFD');\n    text = text_trim(text);\n    text = text.replace(/\\s/g, '');\n    text = text.toLowerCase();\n\n    // Filter out all characters that fall into the Unicode categories listed in the constant UNICODE_FILTER: \"Cc\", \"Cf\", \"Cn\", \"Co\", \"Cs\", \"Mc\", \"Me\", \"Mn\", \"Pc\", \"Pd\", \"Pe\", \"Pf\", \"Pi\", \"Po\", \"Ps\"\n    /*\n    gc ; C                                ; Other                            # Cc | Cf | Cn | Co | Cs\n    gc ; Cc                               ; Control                          ; cntrl\n    gc ; Cf                               ; Format\n    gc ; Cn                               ; Unassigned\n    gc ; Co                               ; Private_Use\n    gc ; Cs                               ; Surrogate\n    */\n    text = text.replace(/\\p{Other}/gu, '');\n\n    /*\n    gc ; M                                ; Mark                             ; Combining_Mark                   # Mc | Me | Mn\n    gc ; Mc                               ; Spacing_Mark\n    gc ; Me                               ; Enclosing_Mark\n    gc ; Mn                               ; Nonspacing_Mark\n    */\n    text = text.replace(/\\p{Mark}/gu, '');\n\n    /*\n    gc ; P                                ; Punctuation                      ; punct                            # Pc | Pd | Pe | Pf | Pi | Po | Ps\n    gc ; Pc                               ; Connector_Punctuation\n    gc ; Pd                               ; Dash_Punctuation\n    gc ; Pe                               ; Close_Punctuation\n    gc ; Pf                               ; Final_Punctuation\n    gc ; Pi                               ; Initial_Punctuation\n    gc ; Po                               ; Other_Punctuation\n    gc ; Ps                               ; Open_Punctuation\n    */\n    text = text.replace(/\\p{Punctuation}/gu, '');\n\n    text = text.normalize('NFKC');\n\n    return text;\n}\n"]}

package/lib/cjs/metacode.d.ts

@@ -0,0 +1,23 @@
+import { IMetaCodeResult } from './constants';
+/**
+ *
+ * Implementation of
+ * https://github.com/iscc/iscc-specs/blob/version-1.0/docs/specification.md#generate-meta-code
+ *
+ * 1 - Verify the requested ISCC version is supported by your implementation.
+ *
+ * @param name
+ * @param description
+ * @param version
+ * @returns IMetaCodeResult
+ */
+export declare function gen_meta_code(name: string, description?: string, meta?: string, bits?: number, version?: number): Promise<IMetaCodeResult>;
+/**
+ *
+ * @param name
+ * @param description
+ * @returns
+ */
+export declare function gen_meta_code_v0(name: string, description?: string, meta?: string, bits?: number): Promise<IMetaCodeResult>;
+export declare function multi_hash_blake3(data: string): Promise<string>;
+export declare function soft_hash_meta_v0(name: string, extra?: string, descJsonFormat?: boolean): Promise<string>;