keepset 0.2.0 → 0.3.0

package/package.json

@@ -1,8 +1,9 @@
 {
   "name": "keepset",
   "description": "Library, CLI and statically generated catalogue for dataset fetch-and-normalise recipes",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "dependencies": {
+    "fflate": "0.8.3",
     "twindex": "0.1.1",
     "zod": "4.6.5"
   },
@@ -40,6 +41,10 @@
       "types": "./dist/index.d.ts",
       "import": "./dist/index.js"
     },
+    "./node": {
+      "types": "./dist/node/fetch-dataset.d.ts",
+      "import": "./dist/node/fetch-dataset.js"
+    },
     "./schemas/*.schema.json": "./schemas/*.schema.json"
   },
   "files": [
@@ -68,7 +73,8 @@
     "url": "git+https://github.com/ExaDev/keepset.git"
   },
   "scripts": {
-    "build": "tsdown && tsx scripts/generate-json-schema.ts",
+    "build": "pnpm run catalogue && tsdown && tsx scripts/generate-json-schema.ts",
+    "catalogue": "tsx scripts/build-catalogue.ts",
     "lint": "eslint . --fix --cache --max-warnings 0",
     "lint:check": "eslint . --cache --max-warnings 0",
     "prepare": "husky",

package/schemas/entry.schema.json

@@ -1 +1 @@
-{"$defs":{"Crosswalk":{"$ref":"https://cdn.jsdelivr.net/npm/twindex@0.1.1/schemas/twindex.schema.json"},"Entry":{"additionalProperties":false,"properties":{"category":{"enum":["graph/v1"],"type":"string"},"citations":{"items":{"additionalProperties":false,"properties":{"doi":{"pattern":"^10\\.\\d{4,9}\\/\\S+$","type":"string"},"text":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["text"],"type":"object"},"type":"array"},"crosswalk":{"anyOf":[{"$ref":"#/$defs/Crosswalk"},{"additionalProperties":false,"properties":{"file":{"type":"string"}},"required":["file"],"type":"object"}]},"description":{"minLength":1,"type":"string"},"domain":{"minLength":1,"type":"string"},"expectedCounts":{"additionalProperties":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"licence":{"oneOf":[{"additionalProperties":false,"properties":{"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9.+-]*$","type":"string"},"kind":{"const":"spdx","type":"string"}},"required":["kind","id"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"custom","type":"string"},"name":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["kind","name","url"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"unrecorded","type":"string"}},"required":["kind"],"type":"object"}]},"name":{"minLength":1,"type":"string"},"properties":{"additionalProperties":{"type":["string","number","boolean","null"]},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"recipe":{"oneOf":[{"additionalProperties":false,"properties":{"archive":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"none","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"gzip","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"zip","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extension":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"}},"required":["extension"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extension":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"}},"required":["extension"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar.gz","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extension":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"}},"required":["extension"],"type":"object"}]}},"required":["kind","member"],"type":"object"}]},"parser":{"oneOf":[{"additionalProperties":false,"properties":{"commentPrefixes":{"items":{"minLength":1,"type":"string"},"type":"array"},"delimiter":{"enum":["whitespace","tab","comma","space"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"edge-list","type":"string"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"sourceColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"targetColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","commentPrefixes","sourceColumn","targetColumn"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"matrix-market","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"gml","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"pajek","type":"string"}},"required":["format","encoding"],"type":"object"}]},"source":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"status":{"const":"fetchable","type":"string"}},"required":["status","source","archive","parser"],"type":"object"},{"additionalProperties":false,"properties":{"reason":{"minLength":1,"type":"string"},"status":{"const":"not-fetchable","type":"string"}},"required":["status","reason"],"type":"object"}]},"sources":{"items":{"additionalProperties":false,"properties":{"retrieved":{"format":"date","pattern":"^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$","type":"string"},"snapshot":{"additionalProperties":false,"properties":{"timestamp":{"pattern":"^\\d{8}(?:\\d{6})?$","type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","timestamp"],"type":"object"},"status":{"enum":["live","dead"],"type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","retrieved","status"],"type":"object"},"minItems":1,"type":"array"},"tags":{"items":{"minLength":1,"type":"string"},"type":"array"}},"required":["id","name","tags","category","sources","licence","citations","recipe","crosswalk","expectedCounts","properties"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.2.0/schemas/entry.schema.json","$ref":"#/$defs/Entry","$schema":"https://json-schema.org/draft/2020-12/schema"}
+{"$defs":{"Crosswalk":{"$ref":"https://cdn.jsdelivr.net/npm/twindex@0.1.1/schemas/twindex.schema.json"},"Entry":{"additionalProperties":false,"properties":{"category":{"enum":["graph/v1"],"type":"string"},"citations":{"items":{"additionalProperties":false,"properties":{"doi":{"pattern":"^10\\.\\d{4,9}\\/\\S+$","type":"string"},"text":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["text"],"type":"object"},"type":"array"},"crosswalk":{"anyOf":[{"$ref":"#/$defs/Crosswalk"},{"additionalProperties":false,"properties":{"file":{"type":"string"}},"required":["file"],"type":"object"}]},"description":{"minLength":1,"type":"string"},"domain":{"minLength":1,"type":"string"},"expectedCounts":{"additionalProperties":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"licence":{"oneOf":[{"additionalProperties":false,"properties":{"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9.+-]*$","type":"string"},"kind":{"const":"spdx","type":"string"}},"required":["kind","id"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"custom","type":"string"},"name":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["kind","name","url"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"unrecorded","type":"string"}},"required":["kind"],"type":"object"}]},"name":{"minLength":1,"type":"string"},"properties":{"additionalProperties":{"type":["string","number","boolean","null"]},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"recipe":{"oneOf":[{"additionalProperties":false,"properties":{"archive":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"none","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"gzip","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"zip","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extension":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"}},"required":["extension"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extension":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"}},"required":["extension"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar.gz","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extension":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"}},"required":["extension"],"type":"object"}]}},"required":["kind","member"],"type":"object"}]},"parser":{"oneOf":[{"additionalProperties":false,"properties":{"commentPrefixes":{"items":{"minLength":1,"type":"string"},"type":"array"},"delimiter":{"enum":["detect","whitespace","tab","comma","space"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"edge-list","type":"string"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"sourceColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"targetColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","commentPrefixes","sourceColumn","targetColumn"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"matrix-market","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"gml","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"pajek","type":"string"}},"required":["format","encoding"],"type":"object"}]},"source":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"status":{"const":"fetchable","type":"string"}},"required":["status","source","archive","parser"],"type":"object"},{"additionalProperties":false,"properties":{"reason":{"minLength":1,"type":"string"},"status":{"const":"not-fetchable","type":"string"}},"required":["status","reason"],"type":"object"}]},"sources":{"items":{"additionalProperties":false,"properties":{"retrieved":{"format":"date","pattern":"^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$","type":"string"},"snapshot":{"additionalProperties":false,"properties":{"timestamp":{"pattern":"^\\d{8}(?:\\d{6})?$","type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","timestamp"],"type":"object"},"status":{"enum":["live","dead"],"type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","retrieved","status"],"type":"object"},"minItems":1,"type":"array"},"tags":{"items":{"minLength":1,"type":"string"},"type":"array"}},"required":["id","name","tags","category","sources","licence","citations","recipe","crosswalk","expectedCounts","properties"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.3.0/schemas/entry.schema.json","$ref":"#/$defs/Entry","$schema":"https://json-schema.org/draft/2020-12/schema"}

package/schemas/manifest.schema.json

@@ -1 +1 @@
-{"$defs":{"Manifest":{"additionalProperties":false,"properties":{"description":{"minLength":1,"type":"string"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"lineage":{"additionalProperties":false,"properties":{"catalogue":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["catalogue","version","repository"],"type":"object"},"name":{"minLength":1,"type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["id","name","description","version","repository"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.2.0/schemas/manifest.schema.json","$ref":"#/$defs/Manifest","$schema":"https://json-schema.org/draft/2020-12/schema"}
+{"$defs":{"Manifest":{"additionalProperties":false,"properties":{"description":{"minLength":1,"type":"string"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"lineage":{"additionalProperties":false,"properties":{"catalogue":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["catalogue","version","repository"],"type":"object"},"name":{"minLength":1,"type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["id","name","description","version","repository"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.3.0/schemas/manifest.schema.json","$ref":"#/$defs/Manifest","$schema":"https://json-schema.org/draft/2020-12/schema"}