@quatrain/okf-ingest-catalog 1.0.1-beta.1 → 1.0.2-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/ai.js +1 -8
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/prompts/catalogEntryPrompt.js +6 -6
- package/dist/types.d.ts +4 -4
- package/package.json +7 -7
- package/src/__test__/catalog.test.ts +136 -136
- package/src/ai.ts +1 -9
- package/src/index.ts +1 -1
- package/src/prompts/catalogEntryPrompt.ts +6 -6
- package/src/types.ts +4 -4
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# @quatrain/okf-ingest-catalog
|
|
2
2
|
|
|
3
|
-
Agnostic ingestion and decomposition engine for encyclopedic catalogs,
|
|
3
|
+
Agnostic ingestion and decomposition engine for encyclopedic catalogs, dictionaries, glossaries and ordered entry collections into Open Knowledge Format (OKF v0.2) atomic entities.
|
|
4
4
|
|
|
5
5
|
## Features
|
|
6
6
|
|
package/dist/ai.js
CHANGED
|
@@ -221,15 +221,8 @@ async function extractCatalogEntryContent(chunk, options) {
|
|
|
221
221
|
}
|
|
222
222
|
body += `\n`;
|
|
223
223
|
}
|
|
224
|
-
if (domainMeta.soils && Array.isArray(domainMeta.soils) && domainMeta.soils.length > 0) {
|
|
225
|
-
body += `## 🌱 Soil & Habitat Characteristics\n\n`;
|
|
226
|
-
for (const s of domainMeta.soils) {
|
|
227
|
-
body += `- ${s}\n`;
|
|
228
|
-
}
|
|
229
|
-
body += `\n`;
|
|
230
|
-
}
|
|
231
224
|
if (properties && properties.length > 0) {
|
|
232
|
-
body += `##
|
|
225
|
+
body += `## ⚙️ Properties & Applications\n\n`;
|
|
233
226
|
for (const p of properties) {
|
|
234
227
|
body += `- ${p}\n`;
|
|
235
228
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* @quatrain/okf-ingest-catalog
|
|
3
3
|
*
|
|
4
4
|
* Agnostic ingestion and decomposition engine for encyclopedic catalogs,
|
|
5
|
-
*
|
|
5
|
+
* dictionaries, glossaries and ordered entry collections into OKF v0.2 atomic entities.
|
|
6
6
|
*/
|
|
7
7
|
export * from './types';
|
|
8
8
|
export * from './detector';
|
package/dist/index.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* @quatrain/okf-ingest-catalog
|
|
4
4
|
*
|
|
5
5
|
* Agnostic ingestion and decomposition engine for encyclopedic catalogs,
|
|
6
|
-
*
|
|
6
|
+
* dictionaries, glossaries and ordered entry collections into OKF v0.2 atomic entities.
|
|
7
7
|
*/
|
|
8
8
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
9
9
|
if (k2 === undefined) k2 = k;
|
|
@@ -9,25 +9,25 @@ const okf_ingest_1 = require("@quatrain/okf-ingest");
|
|
|
9
9
|
function buildCatalogEntryPrompt(params, profile) {
|
|
10
10
|
const { chunk, parentBook } = params;
|
|
11
11
|
const systemRole = profile?.systemRole ||
|
|
12
|
-
'You are an expert
|
|
12
|
+
'You are an expert cataloguer and knowledge engineer specializing in encyclopedic knowledge synthesis and catalog entries according to the Open Knowledge Format (OKF v0.2) specification.';
|
|
13
13
|
const targetLanguages = params.languages || profile?.targetLanguages || ['en'];
|
|
14
14
|
const abstractLines = (0, okf_ingest_1.buildAbstractPromptGuidelines)(targetLanguages);
|
|
15
15
|
const guidelines = [
|
|
16
|
-
'1. "title": Clean primary common or vernacular entry name, optionally followed by
|
|
17
|
-
'2. "scientificName": Canonical
|
|
18
|
-
'3. "family":
|
|
16
|
+
'1. "title": Clean primary common or vernacular entry name, optionally followed by its canonical notation in parentheses (e.g. "Rose quartz (SiO2)").',
|
|
17
|
+
'2. "scientificName": Canonical scientific name, universal designation or official nomenclature.',
|
|
18
|
+
'3. "family": Family, group or class the entry belongs to in its classification.',
|
|
19
19
|
'4. "description": Exactly ONE concise sentence describing the subject, its primary classification, and distinguishing functional characteristics.',
|
|
20
20
|
'5. "language" & "originalLanguage": ISO 639-1 two-letter language code (typically "en" or "fr").',
|
|
21
21
|
`6. "abstracts": Dense, high-value technical synthesis (2 to 3 sentences) describing diagnostic traits and functional properties in the following languages (${targetLanguages.join(', ')}):`,
|
|
22
22
|
...abstractLines,
|
|
23
23
|
`7. "keywords": Standardized indexing keywords (4 to 8 per language) for each configured language (${targetLanguages.join(', ')}).`,
|
|
24
24
|
'8. "tags": Lowercase semantic tags capturing domain taxonomy and parent publication references.',
|
|
25
|
-
'9. "properties": Functional properties,
|
|
25
|
+
'9. "properties": Functional properties, effects or technical applications.',
|
|
26
26
|
];
|
|
27
27
|
if (profile?.promptGuidelines && profile.promptGuidelines.length > 0) {
|
|
28
28
|
guidelines.push(...profile.promptGuidelines);
|
|
29
29
|
}
|
|
30
|
-
guidelines.push('10. "diagrams": Transcribe all charts, comparison matrices, or
|
|
30
|
+
guidelines.push('10. "diagrams": Transcribe all charts, comparison matrices, or descriptive tables into clean Markdown tables or Mermaid diagrams.');
|
|
31
31
|
return `${systemRole}
|
|
32
32
|
|
|
33
33
|
Analyze the following encyclopedic entry extracted from the publication "${parentBook.title}" (Entry #${chunk.sequence}, pages ${chunk.pageRange || 'N/A'}):
|
package/dist/types.d.ts
CHANGED
|
@@ -25,13 +25,13 @@ export interface OkfCatalogEntryMetadata extends OkfFrontmatterV2 {
|
|
|
25
25
|
scientificName?: string;
|
|
26
26
|
/** Vernacular, common or alternative trade names */
|
|
27
27
|
vernacularNames?: string[];
|
|
28
|
-
/**
|
|
28
|
+
/** Family, group or class the entry belongs to */
|
|
29
29
|
family?: string;
|
|
30
|
-
/**
|
|
30
|
+
/** Synonyms and alternative designations */
|
|
31
31
|
synonyms?: string[];
|
|
32
|
-
/**
|
|
32
|
+
/** Diagnostic keys, identifying criteria or mechanisms */
|
|
33
33
|
diagnosticKeys?: string[];
|
|
34
|
-
/**
|
|
34
|
+
/** Functional properties and applications */
|
|
35
35
|
properties?: string[];
|
|
36
36
|
/** Traceable parent book metadata */
|
|
37
37
|
parentBook: CatalogParentBookRef;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@quatrain/okf-ingest-catalog",
|
|
3
|
-
"version": "1.0.
|
|
4
|
-
"description": "Agnostic ingestion and decomposition engine for encyclopedic catalogs,
|
|
3
|
+
"version": "1.0.2-beta.0",
|
|
4
|
+
"description": "Agnostic ingestion and decomposition engine for encyclopedic catalogs, dictionaries, glossaries and ordered entry collections into OKF v0.2 atomic entities",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
7
7
|
"bun": "src/index.ts",
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
],
|
|
14
14
|
"repository": {
|
|
15
15
|
"type": "git",
|
|
16
|
-
"url": "git+https://
|
|
16
|
+
"url": "git+https://git.qtrn.io/Quatrain/Core.git",
|
|
17
17
|
"directory": "packages/okf-ingest-catalog"
|
|
18
18
|
},
|
|
19
19
|
"author": "Quatrain Développement SAS <developers@quatrain.com>",
|
|
@@ -25,11 +25,11 @@
|
|
|
25
25
|
"typescript": "^5.2.2"
|
|
26
26
|
},
|
|
27
27
|
"dependencies": {
|
|
28
|
-
"@quatrain/ai": "^1.1.
|
|
29
|
-
"@quatrain/core": "^1.2.
|
|
28
|
+
"@quatrain/ai": "^1.1.6-beta.0",
|
|
29
|
+
"@quatrain/core": "^1.2.21-beta.5",
|
|
30
30
|
"@quatrain/log": "^1.2.6",
|
|
31
|
-
"@quatrain/okf-ingest": "^1.0.
|
|
32
|
-
"@quatrain/types": "^1.2.
|
|
31
|
+
"@quatrain/okf-ingest": "^1.0.2-beta.1",
|
|
32
|
+
"@quatrain/types": "^1.2.19-beta.4",
|
|
33
33
|
"pdf-parse": "^1.1.1",
|
|
34
34
|
"yaml": "^2.4.0"
|
|
35
35
|
},
|
|
@@ -10,46 +10,46 @@ import { OkfCatalogEntryMetadata } from '../types';
|
|
|
10
10
|
describe('OKF Ingest Catalog Package', () => {
|
|
11
11
|
describe('slugify', () => {
|
|
12
12
|
it('should create URL and directory friendly slugs without diacritics', () => {
|
|
13
|
-
expect(slugify('
|
|
14
|
-
expect(slugify('01 -
|
|
13
|
+
expect(slugify('Quartz rose (Silice cristalline)')).toBe('quartz-rose-silice-cristalline');
|
|
14
|
+
expect(slugify('01 - Épidote de Montréal')).toBe('01-epidote-de-montreal');
|
|
15
15
|
});
|
|
16
16
|
});
|
|
17
17
|
|
|
18
18
|
describe('sliceCatalogEntriesByDescriptors', () => {
|
|
19
19
|
it('should slice text into ordered sequential chunks with parent metadata', () => {
|
|
20
|
-
const rawBookText = `
|
|
20
|
+
const rawBookText = `CATALOGUE DES MINÉRAUX
|
|
21
21
|
Page de garde...
|
|
22
22
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
23
|
+
QUARTZ HYALIN
|
|
24
|
+
SiO2 - Silicates
|
|
25
|
+
Ce minéral forme des cristaux prismatiques transparents...
|
|
26
26
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
27
|
+
CALCITE SPATHIQUE
|
|
28
|
+
CaCO3 - Carbonates
|
|
29
|
+
Ce minéral réagit à l'acide et présente un clivage parfait...
|
|
30
30
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
31
|
+
PYRITE
|
|
32
|
+
FeS2 - Sulfures
|
|
33
|
+
Ce minéral présente un éclat métallique doré...`;
|
|
34
34
|
|
|
35
35
|
const descriptors = [
|
|
36
36
|
{
|
|
37
|
-
title: '
|
|
38
|
-
scientificName: '
|
|
39
|
-
family: '
|
|
40
|
-
startMarker: '
|
|
37
|
+
title: 'Quartz hyalin',
|
|
38
|
+
scientificName: 'SiO2',
|
|
39
|
+
family: 'Silicates',
|
|
40
|
+
startMarker: 'QUARTZ HYALIN',
|
|
41
41
|
},
|
|
42
42
|
{
|
|
43
|
-
title: '
|
|
44
|
-
scientificName: '
|
|
45
|
-
family: '
|
|
46
|
-
startMarker: '
|
|
43
|
+
title: 'Calcite spathique',
|
|
44
|
+
scientificName: 'CaCO3',
|
|
45
|
+
family: 'Carbonates',
|
|
46
|
+
startMarker: 'CALCITE SPATHIQUE',
|
|
47
47
|
},
|
|
48
48
|
{
|
|
49
|
-
title: '
|
|
50
|
-
scientificName: '
|
|
51
|
-
family: '
|
|
52
|
-
startMarker: '
|
|
49
|
+
title: 'Pyrite',
|
|
50
|
+
scientificName: 'FeS2',
|
|
51
|
+
family: 'Sulfures',
|
|
52
|
+
startMarker: 'PYRITE',
|
|
53
53
|
},
|
|
54
54
|
];
|
|
55
55
|
|
|
@@ -57,95 +57,95 @@ Cette plante indique un excès d'azote et un compactage...`;
|
|
|
57
57
|
|
|
58
58
|
expect(chunks.length).toBe(3);
|
|
59
59
|
expect(chunks[0].sequence).toBe(1);
|
|
60
|
-
expect(chunks[0].rawTitle).toBe('
|
|
61
|
-
expect(chunks[0].slug).toBe('001-
|
|
62
|
-
expect(chunks[0].text).toContain('
|
|
63
|
-
expect(chunks[0].text).not.toContain('
|
|
60
|
+
expect(chunks[0].rawTitle).toBe('Quartz hyalin');
|
|
61
|
+
expect(chunks[0].slug).toBe('001-quartz-hyalin');
|
|
62
|
+
expect(chunks[0].text).toContain('cristaux prismatiques transparents');
|
|
63
|
+
expect(chunks[0].text).not.toContain('CALCITE SPATHIQUE');
|
|
64
64
|
|
|
65
65
|
expect(chunks[1].sequence).toBe(2);
|
|
66
|
-
expect(chunks[1].rawTitle).toBe('
|
|
67
|
-
expect(chunks[1].slug).toBe('002-
|
|
68
|
-
expect(chunks[1].text).toContain('
|
|
66
|
+
expect(chunks[1].rawTitle).toBe('Calcite spathique');
|
|
67
|
+
expect(chunks[1].slug).toBe('002-calcite-spathique');
|
|
68
|
+
expect(chunks[1].text).toContain('clivage parfait');
|
|
69
69
|
|
|
70
70
|
expect(chunks[2].sequence).toBe(3);
|
|
71
|
-
expect(chunks[2].rawTitle).toBe('
|
|
72
|
-
expect(chunks[2].slug).toBe('003-
|
|
73
|
-
expect(chunks[2].text).toContain(
|
|
71
|
+
expect(chunks[2].rawTitle).toBe('Pyrite');
|
|
72
|
+
expect(chunks[2].slug).toBe('003-pyrite');
|
|
73
|
+
expect(chunks[2].text).toContain('éclat métallique');
|
|
74
74
|
});
|
|
75
75
|
});
|
|
76
76
|
|
|
77
77
|
describe('detectCatalogEntriesRegex', () => {
|
|
78
78
|
it('should detect entries by regex header and respect minimum length', () => {
|
|
79
|
-
const sampleText = `--- FICHE 1 :
|
|
80
|
-
Texte
|
|
81
|
-
Au moins 100 caractères de contenu
|
|
79
|
+
const sampleText = `--- FICHE 1 : Galena ---
|
|
80
|
+
Texte sur la galène sur plusieurs lignes détaillant sa cristallographie...
|
|
81
|
+
Au moins 100 caractères de contenu descriptif riche pour passer le filtre.
|
|
82
82
|
|
|
83
|
-
--- FICHE 2 :
|
|
84
|
-
Texte
|
|
85
|
-
Au moins 100 caractères de contenu
|
|
83
|
+
--- FICHE 2 : Fluorite ---
|
|
84
|
+
Texte sur la fluorine sur plusieurs lignes avec des descriptions...
|
|
85
|
+
Au moins 100 caractères de contenu technique pour satisfaire la longueur minimale.`;
|
|
86
86
|
|
|
87
87
|
const regex = /^---\s*FICHE\s*\d+\s*:\s*(.+?)\s*---$/gm;
|
|
88
88
|
const chunks = detectCatalogEntriesRegex(sampleText, regex, { minLength: 50 });
|
|
89
89
|
|
|
90
90
|
expect(chunks.length).toBe(2);
|
|
91
91
|
expect(chunks[0].sequence).toBe(1);
|
|
92
|
-
expect(chunks[0].rawTitle).toBe('
|
|
93
|
-
expect(chunks[0].slug).toBe('001-
|
|
92
|
+
expect(chunks[0].rawTitle).toBe('Galena');
|
|
93
|
+
expect(chunks[0].slug).toBe('001-galena');
|
|
94
94
|
expect(chunks[1].sequence).toBe(2);
|
|
95
|
-
expect(chunks[1].rawTitle).toBe('
|
|
96
|
-
expect(chunks[1].slug).toBe('002-
|
|
95
|
+
expect(chunks[1].rawTitle).toBe('Fluorite');
|
|
96
|
+
expect(chunks[1].slug).toBe('002-fluorite');
|
|
97
97
|
});
|
|
98
98
|
});
|
|
99
99
|
|
|
100
100
|
describe('OKF Catalog Entry Serialization', () => {
|
|
101
101
|
it('should serialize and deserialize an entry with sequence and lineage contract faithfully', () => {
|
|
102
102
|
const metadata: OkfCatalogEntryMetadata = {
|
|
103
|
-
type: '
|
|
104
|
-
title: '
|
|
105
|
-
description: '
|
|
106
|
-
tags: ['
|
|
103
|
+
type: 'mineral-profile',
|
|
104
|
+
title: 'Calcite spathique (CaCO3)',
|
|
105
|
+
description: 'Carbonate de calcium au clivage rhomboédrique parfait.',
|
|
106
|
+
tags: ['carbonates', 'clivage', 'catalogue-vol-3'],
|
|
107
107
|
status: 'draft',
|
|
108
108
|
sequence: 42,
|
|
109
109
|
pageRange: '128-130',
|
|
110
|
-
scientificName: '
|
|
111
|
-
family: '
|
|
112
|
-
diagnosticKeys: ['
|
|
110
|
+
scientificName: 'CaCO3',
|
|
111
|
+
family: 'Carbonates',
|
|
112
|
+
diagnosticKeys: ['effervescence à l\'acide', 'clivage parfait', 'dureté 3'],
|
|
113
113
|
parentBook: {
|
|
114
|
-
title: "L'encyclopédie des
|
|
115
|
-
slug: 'encyclopedie-des-
|
|
116
|
-
resource: 'originals/
|
|
114
|
+
title: "L'encyclopédie des minéraux - Volume 3",
|
|
115
|
+
slug: 'encyclopedie-des-mineraux-vol-3',
|
|
116
|
+
resource: 'originals/catalogues/mineraux-vol3.pdf',
|
|
117
117
|
fileHash: 'abcdef1234567890',
|
|
118
118
|
},
|
|
119
119
|
language: 'fr',
|
|
120
120
|
originalLanguage: 'fr',
|
|
121
121
|
abstracts: {
|
|
122
|
-
fr: '
|
|
123
|
-
en: '
|
|
124
|
-
ar: '
|
|
122
|
+
fr: 'Minéral carbonaté effervescent à clivage parfait.',
|
|
123
|
+
en: 'Effervescent carbonate mineral with perfect cleavage.',
|
|
124
|
+
ar: 'معدن كربوني فوار ذو انفصام تام.',
|
|
125
125
|
},
|
|
126
126
|
keywords: {
|
|
127
|
-
fr: ['
|
|
128
|
-
en: ['
|
|
129
|
-
ar: ['
|
|
127
|
+
fr: ['calcite', 'carbonate'],
|
|
128
|
+
en: ['calcite', 'carbonate'],
|
|
129
|
+
ar: ['كالسيت', 'كربونات'],
|
|
130
130
|
},
|
|
131
131
|
};
|
|
132
132
|
|
|
133
|
-
const body = `#
|
|
133
|
+
const body = `# Calcite spathique (CaCO3)\n\nContenu détaillé...`;
|
|
134
134
|
const serialized = serializeOkfDocument(metadata, body);
|
|
135
135
|
|
|
136
|
-
expect(serialized).toContain('type:
|
|
136
|
+
expect(serialized).toContain('type: mineral-profile');
|
|
137
137
|
expect(serialized).toContain('sequence: 42');
|
|
138
|
-
expect(serialized).toContain("title: L'encyclopédie des
|
|
139
|
-
expect(serialized).toContain('scientificName:
|
|
140
|
-
expect(serialized).toContain('fr:
|
|
141
|
-
expect(serialized).toContain('en:
|
|
142
|
-
expect(serialized).toContain('ar:
|
|
138
|
+
expect(serialized).toContain("title: L'encyclopédie des minéraux - Volume 3");
|
|
139
|
+
expect(serialized).toContain('scientificName: CaCO3');
|
|
140
|
+
expect(serialized).toContain('fr: Minéral carbonaté effervescent');
|
|
141
|
+
expect(serialized).toContain('en: Effervescent carbonate mineral');
|
|
142
|
+
expect(serialized).toContain('ar: معدن كربوني فوار');
|
|
143
143
|
|
|
144
144
|
const parsed = parseOkfDocument(serialized);
|
|
145
145
|
const parsedMeta = parsed.metadata as unknown as OkfCatalogEntryMetadata;
|
|
146
146
|
expect(parsedMeta.sequence).toBe(42);
|
|
147
|
-
expect(parsedMeta.scientificName).toBe('
|
|
148
|
-
expect(parsedMeta.parentBook.slug).toBe('encyclopedie-des-
|
|
147
|
+
expect(parsedMeta.scientificName).toBe('CaCO3');
|
|
148
|
+
expect(parsedMeta.parentBook.slug).toBe('encyclopedie-des-mineraux-vol-3');
|
|
149
149
|
expect(parsed.body).toContain('Contenu détaillé...');
|
|
150
150
|
});
|
|
151
151
|
});
|
|
@@ -156,24 +156,24 @@ Au moins 100 caractères de contenu agronomique pour satisfaire la longueur mini
|
|
|
156
156
|
const prompt = buildCatalogEntryPrompt({
|
|
157
157
|
chunk: {
|
|
158
158
|
sequence: 1,
|
|
159
|
-
rawTitle: '
|
|
160
|
-
slug: '001-
|
|
159
|
+
rawTitle: 'Quartz',
|
|
160
|
+
slug: '001-quartz',
|
|
161
161
|
pageRange: '50-51',
|
|
162
|
-
text: '
|
|
162
|
+
text: 'Quartz, famille des silicates...',
|
|
163
163
|
},
|
|
164
164
|
parentBook: {
|
|
165
|
-
title: 'Encyclopédie des
|
|
166
|
-
slug: 'encyclopedie-des-
|
|
165
|
+
title: 'Encyclopédie des minéraux',
|
|
166
|
+
slug: 'encyclopedie-des-mineraux',
|
|
167
167
|
resource: 'test.pdf',
|
|
168
168
|
fileHash: '123456',
|
|
169
169
|
},
|
|
170
170
|
});
|
|
171
171
|
|
|
172
|
-
expect(prompt).toContain('You are an expert
|
|
172
|
+
expect(prompt).toContain('You are an expert cataloguer and knowledge engineer specializing in encyclopedic knowledge synthesis');
|
|
173
173
|
expect(prompt).toContain('1. "title": Clean primary common or vernacular entry name');
|
|
174
|
-
expect(prompt).toContain('2. "scientificName": Canonical
|
|
174
|
+
expect(prompt).toContain('2. "scientificName": Canonical scientific name');
|
|
175
175
|
expect(prompt).toContain('4. "description": Exactly ONE concise sentence');
|
|
176
|
-
expect(prompt).not.toContain('Tu es un ingénieur
|
|
176
|
+
expect(prompt).not.toContain('Tu es un ingénieur');
|
|
177
177
|
});
|
|
178
178
|
|
|
179
179
|
it('should extract catalog entry using a decoupled AbstractAiAdapter and assign standard catalog-entry type', async () => {
|
|
@@ -199,24 +199,24 @@ Au moins 100 caractères de contenu agronomique pour satisfaire la longueur mini
|
|
|
199
199
|
}
|
|
200
200
|
}
|
|
201
201
|
return {
|
|
202
|
-
title: '
|
|
203
|
-
scientificName: '
|
|
204
|
-
family: '
|
|
205
|
-
description: '
|
|
202
|
+
title: 'Quartz hyalin (SiO2)',
|
|
203
|
+
scientificName: 'SiO2',
|
|
204
|
+
family: 'Silicates',
|
|
205
|
+
description: 'Silice cristalline aux prismes hexagonaux.',
|
|
206
206
|
language: 'fr',
|
|
207
207
|
originalLanguage: 'fr',
|
|
208
208
|
abstracts: {
|
|
209
|
-
en: '
|
|
210
|
-
fr: '
|
|
211
|
-
ar: '
|
|
209
|
+
en: 'Crystalline silica with hexagonal prisms.',
|
|
210
|
+
fr: 'Silice cristalline aux prismes hexagonaux.',
|
|
211
|
+
ar: 'سيليكا متبلورة بمناشير سداسية.',
|
|
212
212
|
},
|
|
213
213
|
keywords: {
|
|
214
|
-
en: ['
|
|
215
|
-
fr: ['
|
|
216
|
-
ar: ['
|
|
214
|
+
en: ['quartz', 'silica'],
|
|
215
|
+
fr: ['quartz', 'silice'],
|
|
216
|
+
ar: ['كوارتز', 'سيليكا'],
|
|
217
217
|
},
|
|
218
|
-
tags: ['
|
|
219
|
-
properties: ['
|
|
218
|
+
tags: ['mineralogy', 'silicates'],
|
|
219
|
+
properties: ['piezoelectric', 'optical'],
|
|
220
220
|
};
|
|
221
221
|
}
|
|
222
222
|
}
|
|
@@ -226,16 +226,16 @@ Au moins 100 caractères de contenu agronomique pour satisfaire la longueur mini
|
|
|
226
226
|
const result = await extractCatalogEntryContent(
|
|
227
227
|
{
|
|
228
228
|
sequence: 1,
|
|
229
|
-
rawTitle: '
|
|
230
|
-
slug: '001-
|
|
229
|
+
rawTitle: 'Quartz',
|
|
230
|
+
slug: '001-quartz',
|
|
231
231
|
pageRange: '50-51',
|
|
232
|
-
text: '
|
|
232
|
+
text: 'Quartz hyalin...',
|
|
233
233
|
},
|
|
234
234
|
{
|
|
235
235
|
adapter: mockAdapter,
|
|
236
236
|
parentBook: {
|
|
237
|
-
title: 'Encyclopédie des
|
|
238
|
-
slug: 'encyclopedie-des-
|
|
237
|
+
title: 'Encyclopédie des minéraux',
|
|
238
|
+
slug: 'encyclopedie-des-mineraux',
|
|
239
239
|
resource: 'test.pdf',
|
|
240
240
|
fileHash: '123456',
|
|
241
241
|
},
|
|
@@ -243,11 +243,11 @@ Au moins 100 caractères de contenu agronomique pour satisfaire la longueur mini
|
|
|
243
243
|
);
|
|
244
244
|
|
|
245
245
|
expect(result.metadata.type).toBe('catalog-entry');
|
|
246
|
-
expect(result.metadata.scientificName).toBe('
|
|
247
|
-
expect(result.metadata.family).toBe('
|
|
246
|
+
expect(result.metadata.scientificName).toBe('SiO2');
|
|
247
|
+
expect(result.metadata.family).toBe('Silicates');
|
|
248
248
|
expect(result.metadata.sequence).toBe(1);
|
|
249
249
|
expect(result.usage.total).toBe(120);
|
|
250
|
-
expect(result.body).toContain('**Taxonomy / Classification:** *
|
|
250
|
+
expect(result.body).toContain('**Taxonomy / Classification:** *SiO2* | **Family / Group:** Silicates');
|
|
251
251
|
});
|
|
252
252
|
|
|
253
253
|
it('should support parameterized target languages for catalog entry extraction', async () => {
|
|
@@ -273,22 +273,22 @@ Au moins 100 caractères de contenu agronomique pour satisfaire la longueur mini
|
|
|
273
273
|
}
|
|
274
274
|
}
|
|
275
275
|
return {
|
|
276
|
-
title: '
|
|
277
|
-
scientificName: '
|
|
278
|
-
family: '
|
|
279
|
-
description: '
|
|
276
|
+
title: 'Fluorite (CaF2)',
|
|
277
|
+
scientificName: 'CaF2',
|
|
278
|
+
family: 'Halides',
|
|
279
|
+
description: 'Halide mineral with cubic crystals.',
|
|
280
280
|
language: 'en',
|
|
281
281
|
originalLanguage: 'en',
|
|
282
282
|
abstracts: {
|
|
283
|
-
en: 'High density English summary of
|
|
284
|
-
de: 'Dichte deutsche Zusammenfassung von
|
|
283
|
+
en: 'High density English summary of fluorite.',
|
|
284
|
+
de: 'Dichte deutsche Zusammenfassung von Fluorit.',
|
|
285
285
|
},
|
|
286
286
|
keywords: {
|
|
287
|
-
en: ['
|
|
288
|
-
de: ['
|
|
287
|
+
en: ['fluorite', 'halides'],
|
|
288
|
+
de: ['fluorit', 'halogenide'],
|
|
289
289
|
},
|
|
290
|
-
tags: ['
|
|
291
|
-
properties: ['
|
|
290
|
+
tags: ['halides', 'fluorescence'],
|
|
291
|
+
properties: ['fluorescent', 'flux in metallurgy'],
|
|
292
292
|
};
|
|
293
293
|
}
|
|
294
294
|
}
|
|
@@ -298,19 +298,19 @@ Au moins 100 caractères de contenu agronomique pour satisfaire la longueur mini
|
|
|
298
298
|
const result = await extractCatalogEntryContent(
|
|
299
299
|
{
|
|
300
300
|
sequence: 2,
|
|
301
|
-
rawTitle: '
|
|
302
|
-
slug: '002-
|
|
301
|
+
rawTitle: 'Fluorine',
|
|
302
|
+
slug: '002-fluorine',
|
|
303
303
|
pageRange: '100-102',
|
|
304
|
-
text: '
|
|
304
|
+
text: 'La fluorine en cristaux cubiques...',
|
|
305
305
|
},
|
|
306
306
|
{
|
|
307
307
|
adapter: mockAdapter,
|
|
308
308
|
targetLanguages: ['en', 'de'],
|
|
309
309
|
parentBook: {
|
|
310
|
-
title: 'European
|
|
311
|
-
slug: 'european-
|
|
312
|
-
resource: '
|
|
313
|
-
fileHash: '
|
|
310
|
+
title: 'European Minerals',
|
|
311
|
+
slug: 'european-minerals',
|
|
312
|
+
resource: 'minerals.pdf',
|
|
313
|
+
fileHash: 'minerals123',
|
|
314
314
|
},
|
|
315
315
|
}
|
|
316
316
|
);
|
|
@@ -318,7 +318,7 @@ Au moins 100 caractères de contenu agronomique pour satisfaire la longueur mini
|
|
|
318
318
|
expect(mockAdapter.lastPrompt).toContain('en, de');
|
|
319
319
|
expect(result.metadata.abstracts?.en).toContain('High density English summary');
|
|
320
320
|
expect(result.metadata.abstracts?.de).toContain('Dichte deutsche Zusammenfassung');
|
|
321
|
-
expect(result.metadata.keywords?.de).toEqual(['
|
|
321
|
+
expect(result.metadata.keywords?.de).toEqual(['fluorit', 'halogenide']);
|
|
322
322
|
expect(result.metadata.language).toBe('en');
|
|
323
323
|
});
|
|
324
324
|
|
|
@@ -334,24 +334,24 @@ Au moins 100 caractères de contenu agronomique pour satisfaire la longueur mini
|
|
|
334
334
|
const p = typeof prompt === 'string' ? prompt : JSON.stringify(prompt);
|
|
335
335
|
if (p.includes('scientificName') || p.includes('family')) {
|
|
336
336
|
return {
|
|
337
|
-
title: '
|
|
338
|
-
scientificName: '
|
|
339
|
-
family: '
|
|
340
|
-
description: '
|
|
337
|
+
title: 'Galène',
|
|
338
|
+
scientificName: 'PbS',
|
|
339
|
+
family: 'Sulfures',
|
|
340
|
+
description: 'Sulfure de plomb.',
|
|
341
341
|
language: 'fr',
|
|
342
|
-
abstracts: { en: '
|
|
343
|
-
keywords: { en: ['
|
|
344
|
-
tags: ['
|
|
345
|
-
properties: ['
|
|
342
|
+
abstracts: { en: 'Galena summary' },
|
|
343
|
+
keywords: { en: ['galena'] },
|
|
344
|
+
tags: ['galena', 'sulfides'],
|
|
345
|
+
properties: ['lead ore', 'semiconductor'],
|
|
346
346
|
} as T;
|
|
347
347
|
}
|
|
348
348
|
return {
|
|
349
349
|
title: 'Introduction générale',
|
|
350
350
|
type: 'chapter',
|
|
351
|
-
description: 'Introduction aux principes de la
|
|
352
|
-
category: '
|
|
351
|
+
description: 'Introduction aux principes de la minéralogie.',
|
|
352
|
+
category: 'mineralogy',
|
|
353
353
|
language: 'fr',
|
|
354
|
-
tags: ['
|
|
354
|
+
tags: ['mineralogy', 'classification'],
|
|
355
355
|
} as T;
|
|
356
356
|
}
|
|
357
357
|
}
|
|
@@ -359,38 +359,38 @@ Au moins 100 caractères de contenu agronomique pour satisfaire la longueur mini
|
|
|
359
359
|
const { ingestCatalogMonograph } = await import('../engine');
|
|
360
360
|
const summary = await ingestCatalogMonograph(
|
|
361
361
|
{
|
|
362
|
-
bookTitle: '
|
|
362
|
+
bookTitle: 'Minéraux et Gemmes',
|
|
363
363
|
description: 'Traité encyclopédique de référence.',
|
|
364
|
-
category: '
|
|
364
|
+
category: 'mineralogy',
|
|
365
365
|
authors: ['Dr. A. Schmidt'],
|
|
366
|
-
publisher: '
|
|
366
|
+
publisher: 'Example Press',
|
|
367
367
|
publicationYear: 2024,
|
|
368
368
|
introChapters: [
|
|
369
369
|
{
|
|
370
370
|
index: 1,
|
|
371
371
|
title: 'Introduction générale',
|
|
372
372
|
slug: '01-introduction-generale',
|
|
373
|
-
text: 'Principes généraux de la
|
|
373
|
+
text: 'Principes généraux de la minéralogie...',
|
|
374
374
|
summary: 'Méthodologie générale.',
|
|
375
375
|
},
|
|
376
376
|
],
|
|
377
377
|
entries: [
|
|
378
378
|
{
|
|
379
379
|
sequence: 1,
|
|
380
|
-
rawTitle: '
|
|
381
|
-
slug: '001-
|
|
382
|
-
text: '
|
|
380
|
+
rawTitle: 'Galène',
|
|
381
|
+
slug: '001-galene',
|
|
382
|
+
text: 'PbS. Description complète du minéral...',
|
|
383
383
|
},
|
|
384
384
|
],
|
|
385
385
|
},
|
|
386
386
|
{
|
|
387
|
-
filename: '
|
|
387
|
+
filename: 'mineraux.pdf',
|
|
388
388
|
gitLocalPath: tempDir,
|
|
389
389
|
adapter: new MockCatalogEngineAdapter(),
|
|
390
390
|
}
|
|
391
391
|
);
|
|
392
392
|
|
|
393
|
-
expect(summary.bookSlug).toBe('
|
|
393
|
+
expect(summary.bookSlug).toBe('mineraux-et-gemmes');
|
|
394
394
|
expect(summary.totalEntries).toBe(1);
|
|
395
395
|
expect(summary.totalChapters).toBe(1);
|
|
396
396
|
expect(summary.createdFiles.length).toBe(3); // 1 intro chapter + 1 entry + 1 master index
|
|
@@ -398,7 +398,7 @@ Au moins 100 caractères de contenu agronomique pour satisfaire la longueur mini
|
|
|
398
398
|
const masterPath = path.join(tempDir, summary.masterIndexPath);
|
|
399
399
|
const masterContent = await fs.readFile(masterPath, 'utf-8');
|
|
400
400
|
expect(masterContent).toContain('type: monograph');
|
|
401
|
-
expect(masterContent).toContain('
|
|
401
|
+
expect(masterContent).toContain('PbS');
|
|
402
402
|
expect(masterContent).toContain('01-introduction-generale.md');
|
|
403
403
|
} finally {
|
|
404
404
|
await fs.rm(tempDir, { recursive: true, force: true });
|
package/src/ai.ts
CHANGED
|
@@ -291,16 +291,8 @@ export async function extractCatalogEntryContent(
|
|
|
291
291
|
body += `\n`;
|
|
292
292
|
}
|
|
293
293
|
|
|
294
|
-
if (domainMeta.soils && Array.isArray(domainMeta.soils) && domainMeta.soils.length > 0) {
|
|
295
|
-
body += `## 🌱 Soil & Habitat Characteristics\n\n`;
|
|
296
|
-
for (const s of domainMeta.soils) {
|
|
297
|
-
body += `- ${s}\n`;
|
|
298
|
-
}
|
|
299
|
-
body += `\n`;
|
|
300
|
-
}
|
|
301
|
-
|
|
302
294
|
if (properties && properties.length > 0) {
|
|
303
|
-
body += `##
|
|
295
|
+
body += `## ⚙️ Properties & Applications\n\n`;
|
|
304
296
|
for (const p of properties) {
|
|
305
297
|
body += `- ${p}\n`;
|
|
306
298
|
}
|
package/src/index.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* @quatrain/okf-ingest-catalog
|
|
3
3
|
*
|
|
4
4
|
* Agnostic ingestion and decomposition engine for encyclopedic catalogs,
|
|
5
|
-
*
|
|
5
|
+
* dictionaries, glossaries and ordered entry collections into OKF v0.2 atomic entities.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
export * from './types';
|
|
@@ -22,22 +22,22 @@ export function buildCatalogEntryPrompt(
|
|
|
22
22
|
|
|
23
23
|
const systemRole =
|
|
24
24
|
profile?.systemRole ||
|
|
25
|
-
'You are an expert
|
|
25
|
+
'You are an expert cataloguer and knowledge engineer specializing in encyclopedic knowledge synthesis and catalog entries according to the Open Knowledge Format (OKF v0.2) specification.';
|
|
26
26
|
|
|
27
27
|
const targetLanguages = params.languages || profile?.targetLanguages || ['en'];
|
|
28
28
|
const abstractLines = buildAbstractPromptGuidelines(targetLanguages);
|
|
29
29
|
|
|
30
30
|
const guidelines: string[] = [
|
|
31
|
-
'1. "title": Clean primary common or vernacular entry name, optionally followed by
|
|
32
|
-
'2. "scientificName": Canonical
|
|
33
|
-
'3. "family":
|
|
31
|
+
'1. "title": Clean primary common or vernacular entry name, optionally followed by its canonical notation in parentheses (e.g. "Rose quartz (SiO2)").',
|
|
32
|
+
'2. "scientificName": Canonical scientific name, universal designation or official nomenclature.',
|
|
33
|
+
'3. "family": Family, group or class the entry belongs to in its classification.',
|
|
34
34
|
'4. "description": Exactly ONE concise sentence describing the subject, its primary classification, and distinguishing functional characteristics.',
|
|
35
35
|
'5. "language" & "originalLanguage": ISO 639-1 two-letter language code (typically "en" or "fr").',
|
|
36
36
|
`6. "abstracts": Dense, high-value technical synthesis (2 to 3 sentences) describing diagnostic traits and functional properties in the following languages (${targetLanguages.join(', ')}):`,
|
|
37
37
|
...abstractLines,
|
|
38
38
|
`7. "keywords": Standardized indexing keywords (4 to 8 per language) for each configured language (${targetLanguages.join(', ')}).`,
|
|
39
39
|
'8. "tags": Lowercase semantic tags capturing domain taxonomy and parent publication references.',
|
|
40
|
-
'9. "properties": Functional properties,
|
|
40
|
+
'9. "properties": Functional properties, effects or technical applications.',
|
|
41
41
|
];
|
|
42
42
|
|
|
43
43
|
if (profile?.promptGuidelines && profile.promptGuidelines.length > 0) {
|
|
@@ -45,7 +45,7 @@ export function buildCatalogEntryPrompt(
|
|
|
45
45
|
}
|
|
46
46
|
|
|
47
47
|
guidelines.push(
|
|
48
|
-
'10. "diagrams": Transcribe all charts, comparison matrices, or
|
|
48
|
+
'10. "diagrams": Transcribe all charts, comparison matrices, or descriptive tables into clean Markdown tables or Mermaid diagrams.'
|
|
49
49
|
);
|
|
50
50
|
|
|
51
51
|
return `${systemRole}
|
package/src/types.ts
CHANGED
|
@@ -36,13 +36,13 @@ export interface OkfCatalogEntryMetadata extends OkfFrontmatterV2 {
|
|
|
36
36
|
scientificName?: string;
|
|
37
37
|
/** Vernacular, common or alternative trade names */
|
|
38
38
|
vernacularNames?: string[];
|
|
39
|
-
/**
|
|
39
|
+
/** Family, group or class the entry belongs to */
|
|
40
40
|
family?: string;
|
|
41
|
-
/**
|
|
41
|
+
/** Synonyms and alternative designations */
|
|
42
42
|
synonyms?: string[];
|
|
43
|
-
/**
|
|
43
|
+
/** Diagnostic keys, identifying criteria or mechanisms */
|
|
44
44
|
diagnosticKeys?: string[];
|
|
45
|
-
/**
|
|
45
|
+
/** Functional properties and applications */
|
|
46
46
|
properties?: string[];
|
|
47
47
|
/** Traceable parent book metadata */
|
|
48
48
|
parentBook: CatalogParentBookRef;
|