@larsgw/formica 0.6.8 → 0.7.0

package/CHANGELOG.md

@@ -1,3 +1,22 @@
+# [0.7.0](https://github.com/identification-resources/formica/compare/v0.6.8...v0.7.0) (2025-03-14)
+
+
+### Bug Fixes
+
+* **resources:** improve diff regarding indet lines ([2615437](https://github.com/identification-resources/formica/commit/26154374f50d680c5f3e187cd6a1a5eadbac6b5d))
+
+
+### Features
+
+* **resources:** change syntax of "indet" line ([d53363e](https://github.com/identification-resources/formica/commit/d53363e65bfa77a2eb247999316621f9ca0bff59))
+
+
+### BREAKING CHANGES
+
+* **resources:** "indet" lines now have to be prefixed with "[indet]"
+
+
+
 ## [0.6.8](https://github.com/identification-resources/formica/compare/v0.6.7...v0.6.8) (2025-03-13)
 
 

package/lib/resources/diff-resource.d.ts

@@ -4,4 +4,4 @@ export declare enum ResourceDiffType {
     Modified = "~",
     Unchanged = "="
 }
-export declare function createDiff(a: string, b: string, tokenize?: ResourceDiffTokenizer): ResourceDiffPart[];
+export declare function createDiff(a: string, b: string, tokenize?: ResourceDiffTokenizer): ResourceDiff;

package/lib/resources/diff-resource.js

@@ -68,6 +68,7 @@ function gitTokenize(text) {
     return text.match(/\S+|\n|[\r\t\f\v \u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+/g);
 }
 function createDiff(a, b, tokenize) {
+    var _a, _b;
     if (tokenize === void 0) { tokenize = gitTokenize; }
     var X = tokenize(a.trimEnd());
     var Y = tokenize(b.trimEnd());
@@ -106,7 +107,7 @@ function createDiff(a, b, tokenize) {
         // Start of line
         if (line === null) {
             deletedNewlines = 0;
-            line = { text: '', type: change.type };
+            line = { type: change.type };
         }
         // End of line (could be same token)
         if (change.text === '\n') {
@@ -132,7 +133,8 @@ function createDiff(a, b, tokenize) {
             // Add placeholders for deleted lines
             while (deletedNewlines--) {
                 lines.push({
-                    text: '',
+                    text: undefined,
+                    original: '',
                     type: ResourceDiffType.Deleted
                 });
             }
@@ -144,7 +146,10 @@ function createDiff(a, b, tokenize) {
             line.type = ResourceDiffType.Modified;
         }
         if (change.type !== ResourceDiffType.Deleted) {
-            line.text += change.text;
+            line.text = ((_a = line.text) !== null && _a !== void 0 ? _a : '') + change.text;
+        }
+        if (change.type !== ResourceDiffType.Added) {
+            line.original = ((_b = line.original) !== null && _b !== void 0 ? _b : '') + change.text;
         }
     }
     return lines;

package/lib/resources/parse-text.js

@@ -74,13 +74,6 @@ var TAXONOMIC_STATUS = {
     '+': 'heterotypic synonym',
     '=': 'synonym'
 };
-var INDET_SUFFIXES = new Set([
-    'sp.',
-    'spec.',
-    'indet.',
-    'sp. indet.',
-    'spec. indet.'
-]);
 var RANK_LABELS = {
     'subspecies': 'subsp.',
     'variety': 'var.',
@@ -281,7 +274,7 @@ function parseName(name, rank, parent) {
     item.scientificNameAuthorship = capitalizeAuthors(citation);
     item.taxonRemarks = notes;
     item.taxonRank = rank;
-    if (/[^\p{L}0-9\u{00D7}\- ]/u.test(taxon) && !INDET_SUFFIXES.has(taxon)) {
+    if (/[^\p{L}0-9\u{00D7}\- ]/u.test(taxon)) {
         throw new Error("Taxon name contains unexpected characters: \"".concat(taxon, "\""));
     }
     // Validate names and recompose binomial and trinomial names
@@ -457,6 +450,16 @@ function parseResource(resource) {
     var content = rest.join('');
     return [config, content];
 }
+function getIndentation(line) {
+    return line.match(/^ */)[0].length;
+}
+function isIndetLine(line, indent) {
+    if (indent === undefined) {
+        indent = getIndentation(line);
+    }
+    line = line.slice(indent);
+    return line.startsWith('[indet]');
+}
 function parseResourceContent(content, resource, oldIds) {
     var idBase = "".concat(resource.id, ":");
     var data = resource.taxa;
@@ -465,20 +468,19 @@ function parseResourceContent(content, resource, oldIds) {
     var groupIndent = 0;
     var previousId = '';
     var newIdOffset = Math.max.apply(Math, oldIds);
-    var _loop_1 = function (line, type) {
-        if (type === diff_resource_1.ResourceDiffType.Deleted) {
-            id++;
-            return "continue";
-        }
-        // Do not process "indet" lines further, as they only serve to indicate
-        // that subtaxa are explicitely omitted
-        if (Array.from(INDET_SUFFIXES).some(function (suffix) { return line.endsWith(' ' + suffix); })) {
-            return "continue";
+    for (var _i = 0, content_1 = content; _i < content_1.length; _i++) {
+        var line = content_1[_i];
+        if (line.type === diff_resource_1.ResourceDiffType.Deleted) {
+            // Increase id counter for removed line unless it was an "indet line"
+            if (!isIndetLine(line.original)) {
+                id++;
+            }
+            continue;
         }
-        var lineIndent = line.match(/^ */)[0].length;
+        var lineIndent = getIndentation(line.text);
         if (lineIndent > groupIndent) {
             // Do not count synonyms as parents (unless this is correcting a typo in the synonym)
-            if (data[previousId] && data[previousId].taxonomicStatus === 'accepted' || /^( {2})+> /.test(line)) {
+            if (data[previousId] && data[previousId].taxonomicStatus === 'accepted' || /^( {2})+> /.test(line.text)) {
                 parents.push(previousId);
             }
             else {
@@ -500,17 +502,26 @@ function parseResourceContent(content, resource, oldIds) {
             parents = parents.slice(0, lineIndent / 2);
             groupIndent = lineIndent;
         }
+        // Do not process "indet" lines further, as they only serve to indicate
+        // that subtaxa are explicitely omitted
+        if (isIndetLine(line.text, lineIndent)) {
+            // If the line was previously not and ndet line, increase the id counter
+            if (line.type === diff_resource_1.ResourceDiffType.Modified && !isIndetLine(line.original)) {
+                id++;
+            }
+            continue;
+        }
         var parentId = parents.reduce(function (grandparent, parent) { return parent || grandparent; }, null);
         var parent_1 = parentId === null ? {} : data[parentId];
-        var name_1 = line.slice(groupIndent);
+        var name_1 = line.text.slice(groupIndent);
         var rank = resource.metadata.levels[groupIndent / 2];
         var item = parseName(name_1, rank, parent_1);
         var isSynonym = item.taxonomicStatus !== 'accepted';
         // Add higher classification info
         var itemAsObject = item;
         var parentAsObject = parent_1;
-        for (var _b = 0, DWC_RANKS_1 = DWC_RANKS; _b < DWC_RANKS_1.length; _b++) {
-            var rank_1 = DWC_RANKS_1[_b];
+        for (var _a = 0, DWC_RANKS_1 = DWC_RANKS; _a < DWC_RANKS_1.length; _a++) {
+            var rank_1 = DWC_RANKS_1[_a];
             itemAsObject[rank_1] = undefined;
             if (parentAsObject[rank_1]) {
                 itemAsObject[rank_1] = parentAsObject[rank_1];
@@ -542,10 +553,14 @@ function parseResourceContent(content, resource, oldIds) {
                     parentAsObject[key] = itemAsObject[key];
                 }
             }
-            return "continue";
+            continue;
         }
         // Set identifiers
-        if (type === diff_resource_1.ResourceDiffType.Added) {
+        if (line.type === diff_resource_1.ResourceDiffType.Added) {
+            newIdOffset++;
+            item.scientificNameID = idBase + newIdOffset.toString();
+        }
+        else if (line.type === diff_resource_1.ResourceDiffType.Modified && isIndetLine(line.original)) {
             newIdOffset++;
             item.scientificNameID = idBase + newIdOffset.toString();
         }
@@ -560,10 +575,6 @@ function parseResourceContent(content, resource, oldIds) {
         item.acceptedNameUsage = isSynonym ? parent_1.scientificName : undefined;
         item.collectionCode = idBase.slice(0, -1);
         data[item.scientificNameID] = item;
-    };
-    for (var _i = 0, content_1 = content; _i < content_1.length; _i++) {
-        var _a = content_1[_i], line = _a.text, type = _a.type;
-        _loop_1(line, type);
     }
     return resource;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@larsgw/formica",
-  "version": "0.6.8",
+  "version": "0.7.0",
   "description": "SDK and tools for data from the Library of Identification Resources",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",

package/src/module.d.ts

@@ -95,7 +95,8 @@ interface ResourceHistory {
 type ResourceDiff = ResourceDiffPart[]
 
 interface ResourceDiffPart {
-    text: string,
+    text?: string,
+    original?: string,
     type: ResourceDiffType
 }
 

package/src/resources/diff-resource.ts

@@ -5,7 +5,12 @@ export enum ResourceDiffType {
     Unchanged = '='
 }
 
-function LCS (X: string[], Y: string[]): ResourceDiffPart[] {
+interface DiffPart {
+    text: string,
+    type: ResourceDiffType
+}
+
+function LCS (X: string[], Y: string[]): DiffPart[] {
     const m = X.length
     const n = Y.length
 
@@ -57,7 +62,7 @@ function gitTokenize (text: string): string[] {
     return text.match(/\S+|\n|[\r\t\f\v \u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+/g) as string[]
 }
 
-export function createDiff (a: string, b: string, tokenize: ResourceDiffTokenizer = gitTokenize): ResourceDiffPart[] {
+export function createDiff (a: string, b: string, tokenize: ResourceDiffTokenizer = gitTokenize): ResourceDiff {
     const X = tokenize(a.trimEnd())
     const Y = tokenize(b.trimEnd())
 
@@ -94,7 +99,7 @@ export function createDiff (a: string, b: string, tokenize: ResourceDiffTokenize
 
     // Convert word diff to line diff
     const lines: ResourceDiff = []
-    let line = null
+    let line: ResourceDiffPart|null = null
     let deletedNewlines = 0
     let nextLineNew = false
 
@@ -102,7 +107,7 @@ export function createDiff (a: string, b: string, tokenize: ResourceDiffTokenize
         // Start of line
         if (line === null) {
             deletedNewlines = 0
-            line = { text: '', type: change.type }
+            line = { type: change.type }
         }
 
         // End of line (could be same token)
@@ -132,7 +137,8 @@ export function createDiff (a: string, b: string, tokenize: ResourceDiffTokenize
             // Add placeholders for deleted lines
             while (deletedNewlines--) {
                 lines.push({
-                    text: '',
+                    text: undefined,
+                    original: '',
                     type: ResourceDiffType.Deleted
                 })
             }
@@ -147,7 +153,11 @@ export function createDiff (a: string, b: string, tokenize: ResourceDiffTokenize
         }
 
         if (change.type !== ResourceDiffType.Deleted) {
-            line.text += change.text
+            line.text = (line.text ?? '') + change.text
+        }
+
+        if (change.type !== ResourceDiffType.Added) {
+            line.original = (line.original ?? '') + change.text
         }
     }
 

package/src/resources/parse-text.ts

@@ -66,14 +66,6 @@ const TAXONOMIC_STATUS: Record<string, TaxonStatus> = {
     '=': 'synonym'
 }
 
-const INDET_SUFFIXES = new Set([
-    'sp.',
-    'spec.',
-    'indet.',
-    'sp. indet.',
-    'spec. indet.'
-])
-
 const RANK_LABELS: Record<Rank, string> = {
     'subspecies': 'subsp.',
     'variety': 'var.',
@@ -301,7 +293,7 @@ function parseName (name: string, rank: Rank, parent: WorkingTaxon): WorkingTaxo
     item.taxonRemarks = notes
     item.taxonRank = rank
 
-    if (/[^\p{L}0-9\u{00D7}\- ]/u.test(taxon) && !INDET_SUFFIXES.has(taxon)) {
+    if (/[^\p{L}0-9\u{00D7}\- ]/u.test(taxon)) {
         throw new Error(`Taxon name contains unexpected characters: "${taxon}"`)
     }
 
@@ -488,6 +480,20 @@ function parseResource (resource: string): [ResourceMetadata, string] {
     return [config, content]
 }
 
+function getIndentation (line: string): number {
+    return (line.match(/^ */) as string[])[0].length
+}
+
+function isIndetLine (line: string, indent?: number): boolean {
+    if (indent === undefined) {
+        indent = getIndentation(line)
+    }
+
+    line = line.slice(indent)
+
+    return line.startsWith('[indet]')
+}
+
 function parseResourceContent (content: ResourceDiff, resource: Resource, oldIds: number[]): Resource {
     const idBase = `${resource.id}:`
 
@@ -498,23 +504,19 @@ function parseResourceContent (content: ResourceDiff, resource: Resource, oldIds
     let previousId = ''
     let newIdOffset = Math.max(...oldIds)
 
-    for (const { text: line, type } of content) {
-        if (type === ResourceDiffType.Deleted) {
-            id++
-            continue
-        }
-
-        // Do not process "indet" lines further, as they only serve to indicate
-        // that subtaxa are explicitely omitted
-        if (Array.from(INDET_SUFFIXES).some(suffix => line.endsWith(' ' + suffix))) {
+    for (const line of content) {
+        if (line.type === ResourceDiffType.Deleted) {
+            // Increase id counter for removed line unless it was an "indet line"
+            if (!isIndetLine(line.original as string)) {
+                id++
+            }
             continue
         }
 
-        const lineIndent = (line.match(/^ */) as string[])[0].length
-
+        const lineIndent = getIndentation(line.text as string)
         if (lineIndent > groupIndent) {
             // Do not count synonyms as parents (unless this is correcting a typo in the synonym)
-            if (data[previousId] && data[previousId].taxonomicStatus === 'accepted' || /^( {2})+> /.test(line)) {
+            if (data[previousId] && data[previousId].taxonomicStatus === 'accepted' || /^( {2})+> /.test(line.text as string)) {
                 parents.push(previousId)
             } else {
                 parents.push(null)
@@ -533,10 +535,19 @@ function parseResourceContent (content: ResourceDiff, resource: Resource, oldIds
             groupIndent = lineIndent
         }
 
+        // Do not process "indet" lines further, as they only serve to indicate
+        // that subtaxa are explicitely omitted
+        if (isIndetLine(line.text as string, lineIndent)) {
+            // If the line was previously not and ndet line, increase the id counter
+            if (line.type === ResourceDiffType.Modified && !isIndetLine(line.original as string)) {
+                id++
+            }
+            continue
+        }
+
         const parentId = parents.reduce((grandparent, parent) => parent || grandparent, null)
         const parent = parentId === null ? {} as WorkingTaxon : data[parentId]
-
-        const name = line.slice(groupIndent)
+        const name = (line.text as string).slice(groupIndent)
         const rank = resource.metadata.levels[groupIndent / 2]
         const item = parseName(name, rank, parent)
         const isSynonym = item.taxonomicStatus !== 'accepted'
@@ -581,7 +592,10 @@ function parseResourceContent (content: ResourceDiff, resource: Resource, oldIds
         }
 
         // Set identifiers
-        if (type === ResourceDiffType.Added) {
+        if (line.type === ResourceDiffType.Added) {
+            newIdOffset++
+            item.scientificNameID = idBase + newIdOffset.toString()
+        } else if (line.type === ResourceDiffType.Modified && isIndetLine(line.original as string)) {
             newIdOffset++
             item.scientificNameID = idBase + newIdOffset.toString()
         } else {

package/test/resources.js

@@ -26,9 +26,9 @@ levels: [family, genus, species]
 
     Dolichurus haemorrhous A. Costa, 1886
   Dolichurus A. Costa, 1886
-    Dolichurus indet.
+    [indet]
 Sphecidae A. Costa, 1886
-    Sphecidae indet.
+    [indet]
 `, 'T1')
         assert.deepStrictEqual(Object.values(resource.taxa).map(taxon => taxon.scientificNameAuthorship), Array(3).fill('A. Costa, 1886'))
     })
@@ -86,7 +86,7 @@ levels: [genus, species]
 ---
 
 Drymus
-  Unknown sp.
+  [indet]
 `, 'T1')
         assert.strictEqual(resource.taxa['T1:1:1'].scientificName, 'Drymus')
     })
@@ -156,7 +156,7 @@ levels: [genus, species]
 ---
 
 x Triticosecale
-  Triticosecale indet.
+  [indet]
 x Festulpia
   Festuca_rubra x Vulpia_bromoides
 `, 'T1')