@larsgw/formica 0.6.1 → 0.6.2

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+## [0.6.2](https://github.com/identification-resources/formica/compare/v0.6.1...v0.6.2) (2024-04-08)
+
+
+### Bug Fixes
+
+* **catalog:** include all fields in tabular export ([c4e47bb](https://github.com/identification-resources/formica/commit/c4e47bbdf31c8f589e35c897c8207eb4dd979461))
+
+
+### Features
+
+* **catalog:** add script to de-duplicate links ([b719348](https://github.com/identification-resources/formica/commit/b719348635e1b35ee80758c0cdd9938625d79ebf))
+
+
+
 ## [0.6.1](https://github.com/identification-resources/formica/compare/v0.6.0...v0.6.1) (2024-03-20)
 
 

package/lib/bin/clean-links.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/lib/bin/clean-links.js

@@ -0,0 +1,170 @@
+#!/usr/bin/env node
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+var fs_1 = require("fs");
+var path = require("path");
+var util = require("util");
+var index_1 = require("../index");
+var LINKS = {
+    author: { sheet: 'authors', cleanField: 'main_full_name' },
+    publisher: { sheet: 'publishers', cleanField: 'full_name' },
+};
+function replaceLink(value, data, cleanField) {
+    var _a;
+    var entity = data.get(value);
+    return (_a = entity.get(cleanField)) !== null && _a !== void 0 ? _a : value;
+}
+function save(sheet, path) {
+    return fs_1.promises.writeFile(path, index_1.csv.formatCsv(sheet.toTable()));
+}
+function main() {
+    return __awaiter(this, void 0, void 0, function () {
+        var args, files, _i, _a, arg, filePath, file, sheet, _loop_1, _b, _c, _d, _e, field;
+        return __generator(this, function (_f) {
+            switch (_f.label) {
+                case 0:
+                    args = util.parseArgs({ allowPositionals: true });
+                    files = {};
+                    _i = 0, _a = args.positionals;
+                    _f.label = 1;
+                case 1:
+                    if (!(_i < _a.length)) return [3 /*break*/, 4];
+                    arg = _a[_i];
+                    filePath = path.resolve(arg);
+                    return [4 /*yield*/, fs_1.promises.readFile(filePath, 'utf8')];
+                case 2:
+                    file = _f.sent();
+                    sheet = path.basename(filePath, '.csv');
+                    files[sheet] = {
+                        path: filePath,
+                        sheet: index_1.catalog.loadData(file, sheet),
+                    };
+                    _f.label = 3;
+                case 3:
+                    _i++;
+                    return [3 /*break*/, 1];
+                case 4:
+                    if (!files.catalog) {
+                        throw new Error('Catalog file must be provided');
+                    }
+                    _loop_1 = function (field) {
+                        var _g, sheet, cleanField, indexField, used, _h, _j, work, value, entities, _k, _l, entity, oldKey, newKey;
+                        return __generator(this, function (_m) {
+                            switch (_m.label) {
+                                case 0:
+                                    _g = LINKS[field], sheet = _g.sheet, cleanField = _g.cleanField;
+                                    indexField = index_1.catalog.getTypeInfo(sheet)[1];
+                                    used = new Set();
+                                    if (!files[sheet]) {
+                                        return [2 /*return*/, "continue"];
+                                    }
+                                    // Update works
+                                    for (_h = 0, _j = files.catalog.sheet.entities; _h < _j.length; _h++) {
+                                        work = _j[_h];
+                                        value = work.get(field);
+                                        if (value === undefined) {
+                                            continue;
+                                        }
+                                        else if (Array.isArray(value)) {
+                                            work.fields[field] = value.map(function (value) {
+                                                value = replaceLink(value, files[sheet].sheet, cleanField);
+                                                used.add(value);
+                                                return value;
+                                            });
+                                        }
+                                        else {
+                                            work.fields[field] = replaceLink(value, files[sheet].sheet, cleanField);
+                                            used.add(work.fields[field]);
+                                        }
+                                    }
+                                    entities = [];
+                                    for (_k = 0, _l = files[sheet].sheet.entities; _k < _l.length; _k++) {
+                                        entity = _l[_k];
+                                        oldKey = entity.get(indexField);
+                                        if (entity.has(cleanField)) {
+                                            entity.fields[indexField] = entity.get(cleanField);
+                                            delete entity.fields[cleanField];
+                                        }
+                                        newKey = entity.get(indexField);
+                                        if (used.has(newKey) && (newKey === oldKey || !files[sheet].sheet.index[newKey])) {
+                                            entities.push(entity);
+                                        }
+                                    }
+                                    // Save new file
+                                    return [4 /*yield*/, save(new index_1.catalog.Entities(entities, indexField), files[sheet].path)];
+                                case 1:
+                                    // Save new file
+                                    _m.sent();
+                                    return [2 /*return*/];
+                            }
+                        });
+                    };
+                    _b = LINKS;
+                    _c = [];
+                    for (_d in _b)
+                        _c.push(_d);
+                    _e = 0;
+                    _f.label = 5;
+                case 5:
+                    if (!(_e < _c.length)) return [3 /*break*/, 8];
+                    _d = _c[_e];
+                    if (!(_d in _b)) return [3 /*break*/, 7];
+                    field = _d;
+                    return [5 /*yield**/, _loop_1(field)];
+                case 6:
+                    _f.sent();
+                    _f.label = 7;
+                case 7:
+                    _e++;
+                    return [3 /*break*/, 5];
+                case 8: 
+                // Save catalog
+                return [4 /*yield*/, save(files.catalog.sheet, files.catalog.path)];
+                case 9:
+                    // Save catalog
+                    _f.sent();
+                    return [2 /*return*/];
+            }
+        });
+    });
+}
+main().catch(function (error) {
+    console.error(error);
+    process.exit(1);
+});

package/lib/catalog/entities.js

@@ -97,7 +97,7 @@ var Entities = /** @class */ (function () {
         if (this.entities.length === 0) {
             return [[]];
         }
-        var header = Object.keys(this.entities[0].fields);
+        var header = Object.keys(this.entities[0].schema);
         var table = this.entities.map(function (entity) {
             return header.map(function (field) {
                 var value = entity.get(field) || '';

package/lib/catalog/index.d.ts

@@ -1,4 +1,5 @@
 import { Entities } from './entities';
 import { Entity } from './entity';
-export { Entities, Entity };
+declare function getTypeInfo(type: string): [typeof Entity, string];
+export { Entities, Entity, getTypeInfo };
 export declare function loadData(file: string, type: string): Entities;

package/lib/catalog/index.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadData = exports.Entity = exports.Entities = void 0;
+exports.loadData = exports.getTypeInfo = exports.Entity = exports.Entities = void 0;
 var entities_1 = require("./entities");
 Object.defineProperty(exports, "Entities", { enumerable: true, get: function () { return entities_1.Entities; } });
 var entity_1 = require("./entity");
@@ -16,6 +16,7 @@ function getTypeInfo(type) {
         default: throw new TypeError("Unknown type \"".concat(type, "\""));
     }
 }
+exports.getTypeInfo = getTypeInfo;
 function loadData(file, type) {
     var _a = getTypeInfo(type), subClass = _a[0], indexField = _a[1];
     var entities = [];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@larsgw/formica",
-  "version": "0.6.1",
+  "version": "0.6.2",
   "description": "SDK and tools for data from the Library of Identification Resources",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
@@ -8,7 +8,8 @@
     "loir-validate-catalog": "./lib/bin/validate-catalog.js",
     "loir-validate-resources": "./lib/bin/validate-resources-text.js",
     "loir-resources-process": "./lib/bin/process-resources.js",
-    "loir-resources-index": "./lib/bin/process-resources-index.js"
+    "loir-resources-index": "./lib/bin/process-resources-index.js",
+    "loir-catalog-clean-links": "./lib/bin/clean-links.js"
   },
   "scripts": {
     "test": "node --test --test-reporter spec",

package/src/bin/clean-links.ts

@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+
+import { promises as fs } from 'fs'
+import * as path from 'path'
+import * as util from 'util'
+
+import { catalog, csv } from '../index'
+
+const LINKS: Record<string, { sheet: string, cleanField: string }> = {
+    author: { sheet: 'authors', cleanField: 'main_full_name' },
+    publisher: { sheet: 'publishers', cleanField: 'full_name' },
+}
+
+function replaceLink (value: string, data: catalog.Entities, cleanField: string): string {
+    const entity = data.get(value)
+    return entity.get(cleanField) as string ?? value
+}
+
+function save (sheet: catalog.Entities, path: string): Promise<void> {
+    return fs.writeFile(path, csv.formatCsv(sheet.toTable()))
+}
+
+async function main (): Promise<void> {
+    const args = util.parseArgs({ allowPositionals: true })
+
+    const files: Record<string, { path: string, sheet: catalog.Entities }> = {}
+    for (const arg of args.positionals) {
+        const filePath = path.resolve(arg)
+        const file = await fs.readFile(filePath, 'utf8')
+        const sheet = path.basename(filePath, '.csv')
+        files[sheet] = {
+            path: filePath,
+            sheet: catalog.loadData(file, sheet),
+        }
+    }
+
+    if (!files.catalog) {
+        throw new Error('Catalog file must be provided')
+    }
+
+    for (const field in LINKS) {
+        const { sheet, cleanField } = LINKS[field]
+        const indexField = catalog.getTypeInfo(sheet)[1]
+        const used = new Set()
+
+        if (!files[sheet]) {
+            continue
+        }
+
+        // Update works
+        for (const work of files.catalog.sheet.entities) {
+            const value = work.get(field)
+
+            if (value === undefined) {
+                continue
+            } else if (Array.isArray(value)) {
+                work.fields[field] = value.map(value => {
+                    value = replaceLink(value, files[sheet].sheet, cleanField)
+                    used.add(value)
+                    return value
+                })
+            } else {
+                work.fields[field] = replaceLink(value, files[sheet].sheet, cleanField)
+                used.add(work.fields[field])
+            }
+        }
+
+        // Filter unused values
+        const entities = []
+        for (const entity of files[sheet].sheet.entities) {
+            const oldKey = entity.get(indexField) as string
+
+            if (entity.has(cleanField)) {
+                entity.fields[indexField] = entity.get(cleanField) as Value
+                delete entity.fields[cleanField]
+            }
+
+            const newKey = entity.get(indexField) as string
+
+            if (used.has(newKey) && (newKey === oldKey || !files[sheet].sheet.index[newKey])) {
+                entities.push(entity)
+            }
+        }
+
+        // Save new file
+        await save(new catalog.Entities(entities, indexField), files[sheet].path)
+    }
+
+    // Save catalog
+    await save(files.catalog.sheet, files.catalog.path)
+}
+
+main().catch((error: Error) => {
+    console.error(error)
+    process.exit(1)
+})

package/src/catalog/entities.ts

@@ -50,7 +50,7 @@ export class Entities {
         if (this.entities.length === 0) {
             return [[]]
         }
-        const header = Object.keys(this.entities[0].fields)
+        const header = Object.keys(this.entities[0].schema)
         const table = this.entities.map((entity: Entity) => {
             return header.map((field: string): string => {
                 const value = entity.get(field) || ''

package/src/catalog/index.ts

@@ -13,7 +13,7 @@ function getTypeInfo (type: string): [typeof Entity, string] {
     }
 }
 
-export { Entities, Entity }
+export { Entities, Entity, getTypeInfo }
 
 export function loadData (file: string, type: string): Entities {
     const [subClass, indexField] = getTypeInfo(type)