@quatrain/okf-ingest-catalog 0.0.0-stage → 1.0.1-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,3 +1,28 @@
1
- # Temporary Holding Version
1
+ # @quatrain/okf-ingest-catalog
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ Agnostic ingestion and decomposition engine for encyclopedic catalogs, plant floras, dictionaries, pharmacopeias, and ordered entry collections into Open Knowledge Format (OKF v0.2) atomic entities.
4
+
5
+ ## Features
6
+
7
+ - **Agnostic Catalog Slicing**: Splits massive encyclopedic catalogs by canonical index descriptors (`sliceCatalogEntriesByDescriptors`) or structured regex patterns (`detectCatalogEntriesRegex`).
8
+ - **Sequential Ordering & Traceability**: Automatically numbers records sequentially (`sequence: 1, 2, ... N`), preserves original page ranges (`pageRange`), and captures canonical Latin/scientific binomials (`scientificName`) and families (`family`).
9
+ - **Lineage Contract**: Injects parent book provenance (`parentBook: { title, slug, resource, fileHash }`) into every child record.
10
+ - **Model Decoupled**: Delegates all structured AI generation via `@quatrain/ai` adapters (`AbstractAiAdapter`).
11
+ - **Configurable Multilingual Targets**: Generates high-density abstracts and keywords in requested languages (`targetLanguages: ['en', 'fr', 'ar']`).
12
+ - **Master Monograph Index Generation**: Automatically creates a root `index.md` featuring methodological chapters and a Markdown table referencing every entry.
13
+
14
+ ## Installation
15
+
16
+ ```bash
17
+ yarn add @quatrain/okf-ingest-catalog @quatrain/okf-ingest @quatrain/ai
18
+ # or
19
+ bun add @quatrain/okf-ingest-catalog @quatrain/okf-ingest @quatrain/ai
20
+ ```
21
+
22
+ ## Documentation
23
+
24
+ See the comprehensive [HOWTO Guide](./HOWTO.md) for step-by-step instructions and code examples.
25
+
26
+ ## License
27
+
28
+ AGPL-3.0-only
package/dist/ai.d.ts ADDED
@@ -0,0 +1,38 @@
1
+ import { AbstractAiAdapter } from '@quatrain/ai';
2
+ import { AiStructuredRunner, DomainTaxonomyProfile, OkfDocumentType, OkfJsonSchema, OkfJsonSchemaProperty, OkfTokenUsage } from '@quatrain/okf-ingest';
3
+ import { CatalogEntryChunk, CatalogParentBookRef, OkfCatalogEntryMetadata } from './types';
4
+ /**
5
+ * Universal JSON Schema properties for catalog and encyclopedic entries.
6
+ */
7
+ export declare const CATALOG_BASE_PROPERTIES: Record<string, OkfJsonSchemaProperty>;
8
+ /**
9
+ * Builds a JSON Schema for structured catalog entry extraction,
10
+ * merging universal catalog properties with profile domain extensions and configured target languages.
11
+ */
12
+ export declare function buildCatalogAiSchema(profile?: DomainTaxonomyProfile, languages?: string[]): OkfJsonSchema;
13
+ export declare const CATALOG_ENTRY_AI_SCHEMA: OkfJsonSchema;
14
+ export interface EntryExtractionOptions {
15
+ apiKey?: string;
16
+ adapter?: AbstractAiAdapter;
17
+ runner?: AiStructuredRunner;
18
+ targetLanguages?: string[];
19
+ languages?: string[];
20
+ parentBook: CatalogParentBookRef;
21
+ entryType?: OkfDocumentType;
22
+ model?: string;
23
+ defaultCategory?: string;
24
+ soa?: string;
25
+ revision?: string;
26
+ taxonomyProfile?: DomainTaxonomyProfile;
27
+ }
28
+ /**
29
+ * Extracts and enriches a single encyclopedic/catalog entry.
30
+ * Model-agnostic and delegates strictly through @quatrain/ai adapters.
31
+ */
32
+ export declare function extractCatalogEntryContent(chunk: CatalogEntryChunk, options: EntryExtractionOptions): Promise<{
33
+ metadata: OkfCatalogEntryMetadata;
34
+ body: string;
35
+ usage: OkfTokenUsage;
36
+ diagramsTranscribed: number;
37
+ tablesTranscribed: number;
38
+ }>;
package/dist/ai.js ADDED
@@ -0,0 +1,255 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.CATALOG_ENTRY_AI_SCHEMA = exports.CATALOG_BASE_PROPERTIES = void 0;
4
+ exports.buildCatalogAiSchema = buildCatalogAiSchema;
5
+ exports.extractCatalogEntryContent = extractCatalogEntryContent;
6
+ const okf_ingest_1 = require("@quatrain/okf-ingest");
7
+ const catalogEntryPrompt_1 = require("./prompts/catalogEntryPrompt");
8
+ /**
9
+ * Universal JSON Schema properties for catalog and encyclopedic entries.
10
+ */
11
+ exports.CATALOG_BASE_PROPERTIES = {
12
+ title: { type: 'STRING' },
13
+ scientificName: { type: 'STRING' },
14
+ family: { type: 'STRING' },
15
+ description: { type: 'STRING' },
16
+ category: { type: 'STRING' },
17
+ language: { type: 'STRING' },
18
+ originalLanguage: { type: 'STRING' },
19
+ tags: {
20
+ type: 'ARRAY',
21
+ items: { type: 'STRING' },
22
+ },
23
+ properties: {
24
+ type: 'ARRAY',
25
+ items: { type: 'STRING' },
26
+ },
27
+ diagnosticKeys: {
28
+ type: 'ARRAY',
29
+ items: { type: 'STRING' },
30
+ },
31
+ diagrams: okf_ingest_1.OKF_DIAGRAMS_SCHEMA_PROPERTY,
32
+ };
33
+ /**
34
+ * Builds a JSON Schema for structured catalog entry extraction,
35
+ * merging universal catalog properties with profile domain extensions and configured target languages.
36
+ */
37
+ function buildCatalogAiSchema(profile, languages = ['en']) {
38
+ const { abstractProps, keywordProps } = (0, okf_ingest_1.buildMultilingualSchemaProperties)(languages);
39
+ const properties = {
40
+ ...exports.CATALOG_BASE_PROPERTIES,
41
+ abstracts: {
42
+ type: 'OBJECT',
43
+ properties: abstractProps,
44
+ required: languages,
45
+ },
46
+ keywords: {
47
+ type: 'OBJECT',
48
+ properties: keywordProps,
49
+ required: languages,
50
+ },
51
+ ...profile?.schemaProperties,
52
+ };
53
+ return {
54
+ type: 'OBJECT',
55
+ properties,
56
+ required: [
57
+ 'title',
58
+ 'scientificName',
59
+ 'family',
60
+ 'description',
61
+ 'language',
62
+ 'abstracts',
63
+ 'keywords',
64
+ 'tags',
65
+ ],
66
+ };
67
+ }
68
+ exports.CATALOG_ENTRY_AI_SCHEMA = buildCatalogAiSchema();
69
+ /**
70
+ * Extracts and enriches a single encyclopedic/catalog entry.
71
+ * Model-agnostic and delegates strictly through @quatrain/ai adapters.
72
+ */
73
+ async function extractCatalogEntryContent(chunk, options) {
74
+ const adapter = (0, okf_ingest_1.resolveAiAdapter)({ adapter: options.adapter, apiKey: options.apiKey });
75
+ const profile = options.taxonomyProfile || new okf_ingest_1.GenericDomainProfile();
76
+ const targetLanguages = options.targetLanguages || options.languages || profile.targetLanguages || ['en'];
77
+ const schema = buildCatalogAiSchema(profile, targetLanguages);
78
+ const model = options.model || process.env.GEMINI_MODEL || 'gemini-2.5-flash';
79
+ const prompt = (0, catalogEntryPrompt_1.buildCatalogEntryPrompt)({
80
+ chunk,
81
+ parentBook: options.parentBook,
82
+ languages: targetLanguages,
83
+ }, profile);
84
+ let rawUsage = null;
85
+ const rawRecord = (await adapter.generateStructured(prompt, schema, {
86
+ model,
87
+ onUsage: (u) => {
88
+ if (typeof u === 'object' && u !== null) {
89
+ rawUsage = u;
90
+ }
91
+ },
92
+ }));
93
+ // Normalize if model mirrored schema metadata structure { type: 'OBJECT', properties: { ... } }
94
+ const parsed = rawRecord.properties && typeof rawRecord.properties === 'object' && !rawRecord.title
95
+ ? rawRecord.properties
96
+ : rawRecord;
97
+ let usage;
98
+ if (rawUsage !== null) {
99
+ usage = (0, okf_ingest_1.calculateTokenCost)(rawUsage, model);
100
+ }
101
+ else {
102
+ const estPrompt = Math.ceil(prompt.length / 4);
103
+ const estOutput = Math.ceil(JSON.stringify(parsed).length / 4);
104
+ usage = (0, okf_ingest_1.calculateTokenCost)({ promptTokenCount: estPrompt, candidatesTokenCount: estOutput, totalTokenCount: estPrompt + estOutput }, model);
105
+ }
106
+ const title = typeof parsed.title === 'string' ? parsed.title : chunk.rawTitle;
107
+ const scientificName = typeof parsed.scientificName === 'string' ? parsed.scientificName : chunk.scientificName;
108
+ const family = typeof parsed.family === 'string' ? parsed.family : chunk.family;
109
+ const description = typeof parsed.description === 'string' ? parsed.description : `Catalog entry for ${title}.`;
110
+ const category = typeof parsed.category === 'string'
111
+ ? parsed.category
112
+ : (options.defaultCategory || profile.defaultCategory || 'catalog');
113
+ const language = typeof parsed.language === 'string' ? parsed.language : (targetLanguages[0] || 'en');
114
+ const originalLanguage = typeof parsed.originalLanguage === 'string'
115
+ ? parsed.originalLanguage
116
+ : language;
117
+ const abstracts = {};
118
+ if (typeof parsed.abstracts === 'object' && parsed.abstracts !== null) {
119
+ const parsedAbs = parsed.abstracts;
120
+ for (const lang of targetLanguages) {
121
+ const val = Reflect.get(parsedAbs, lang);
122
+ if (typeof val === 'string') {
123
+ Reflect.set(abstracts, lang, val);
124
+ }
125
+ }
126
+ }
127
+ if (Object.keys(abstracts).length === 0) {
128
+ for (const lang of targetLanguages) {
129
+ Reflect.set(abstracts, lang, description);
130
+ }
131
+ }
132
+ const keywords = {};
133
+ if (typeof parsed.keywords === 'object' && parsed.keywords !== null) {
134
+ const parsedKw = parsed.keywords;
135
+ for (const lang of targetLanguages) {
136
+ const val = Reflect.get(parsedKw, lang);
137
+ if (Array.isArray(val)) {
138
+ Reflect.set(keywords, lang, val);
139
+ }
140
+ }
141
+ }
142
+ if (Object.keys(keywords).length === 0) {
143
+ for (const lang of targetLanguages) {
144
+ Reflect.set(keywords, lang, [title]);
145
+ }
146
+ }
147
+ const tags = Array.isArray(parsed.tags) ? parsed.tags : [];
148
+ const properties = Array.isArray(parsed.properties) ? parsed.properties : undefined;
149
+ const diagnosticKeys = Array.isArray(parsed.diagnosticKeys) ? parsed.diagnosticKeys : undefined;
150
+ // Process diagrams and Markdown tables
151
+ let diagramsCount = 0;
152
+ let tablesCount = 0;
153
+ const diagramsList = [];
154
+ if (Array.isArray(parsed.diagrams)) {
155
+ for (const item of parsed.diagrams) {
156
+ if (typeof item === 'object' && item !== null) {
157
+ const diag = item;
158
+ const diagType = String(diag.type || 'table');
159
+ const diagTitle = String(diag.title || 'Illustration');
160
+ const diagContent = String(diag.content || '').trim();
161
+ const diagExplanation = String(diag.explanation || '').trim();
162
+ if (diagType === 'mermaid' && diagContent) {
163
+ diagramsCount++;
164
+ diagramsList.push(`### ${diagTitle}\n\n\`\`\`mermaid\n${diagContent}\n\`\`\`\n\n*${diagExplanation}*\n`);
165
+ }
166
+ else if (diagType === 'table' && diagContent) {
167
+ tablesCount++;
168
+ diagramsList.push(`### ${diagTitle}\n\n${diagContent}\n\n*${diagExplanation}*\n`);
169
+ }
170
+ }
171
+ }
172
+ }
173
+ const domainMeta = profile.extractDomainMetadata ? profile.extractDomainMetadata(parsed) : {};
174
+ const metadata = {
175
+ type: options.entryType || 'catalog-entry',
176
+ title,
177
+ description,
178
+ tags: Array.from(new Set([...tags, options.parentBook.slug])),
179
+ status: 'draft',
180
+ generated: {
181
+ by: `quatrain/okf-ingest-catalog (${model})`,
182
+ at: new Date().toISOString(),
183
+ tokens: usage,
184
+ },
185
+ sources: [
186
+ {
187
+ id: 'parent-book',
188
+ resource: options.parentBook.resource,
189
+ title: options.parentBook.title,
190
+ fileHash: options.parentBook.fileHash,
191
+ },
192
+ ],
193
+ soa: options.soa || 'quatrain/knowledge',
194
+ revision: options.revision,
195
+ category,
196
+ language,
197
+ originalLanguage,
198
+ abstracts,
199
+ keywords,
200
+ sequence: chunk.sequence,
201
+ pageRange: chunk.pageRange,
202
+ scientificName,
203
+ family,
204
+ diagnosticKeys,
205
+ properties,
206
+ parentBook: options.parentBook,
207
+ ...domainMeta,
208
+ };
209
+ // Format rich, structured Markdown body
210
+ let body = `# ${title}\n\n`;
211
+ if (scientificName && family) {
212
+ body += `**Taxonomy / Classification:** *${scientificName}* | **Family / Group:** ${family}\n\n`;
213
+ }
214
+ if (chunk.pageRange) {
215
+ body += `> Extracted from: [${options.parentBook.title}](../index.md) (pages ${chunk.pageRange})\n\n`;
216
+ }
217
+ if (diagnosticKeys && diagnosticKeys.length > 0) {
218
+ body += `## 🔬 Key Diagnostic Criteria & Indicators\n\n`;
219
+ for (const key of diagnosticKeys) {
220
+ body += `- **${key}**\n`;
221
+ }
222
+ body += `\n`;
223
+ }
224
+ if (domainMeta.soils && Array.isArray(domainMeta.soils) && domainMeta.soils.length > 0) {
225
+ body += `## 🌱 Soil & Habitat Characteristics\n\n`;
226
+ for (const s of domainMeta.soils) {
227
+ body += `- ${s}\n`;
228
+ }
229
+ body += `\n`;
230
+ }
231
+ if (properties && properties.length > 0) {
232
+ body += `## 🌿 Properties & Applications\n\n`;
233
+ for (const p of properties) {
234
+ body += `- ${p}\n`;
235
+ }
236
+ body += `\n`;
237
+ }
238
+ if (typeof profile.renderMarkdownSections === 'function') {
239
+ const customSections = profile.renderMarkdownSections(domainMeta);
240
+ if (customSections) {
241
+ body += `${customSections}\n\n`;
242
+ }
243
+ }
244
+ if (diagramsList.length > 0) {
245
+ body += `## 📊 Analytical Data & Diagrams\n\n${diagramsList.join('\n')}\n`;
246
+ }
247
+ body += `## 📖 Detailed Entry Description & Source Text\n\n${chunk.text.trim()}\n`;
248
+ return {
249
+ metadata,
250
+ body,
251
+ usage,
252
+ diagramsTranscribed: diagramsCount,
253
+ tablesTranscribed: tablesCount,
254
+ };
255
+ }
@@ -0,0 +1,22 @@
1
+ import { CatalogEntryChunk } from './types';
2
+ /**
3
+ * Normalizes text into a clean lowercase URL/filesystem slug.
4
+ */
5
+ export declare function slugify(text: string): string;
6
+ export interface CatalogEntryDescriptor {
7
+ title: string;
8
+ scientificName?: string;
9
+ family?: string;
10
+ pageRange?: string;
11
+ startMarker: string;
12
+ }
13
+ /**
14
+ * Slices raw monograph text into ordered catalog entry chunks based on explicit markers or titles.
15
+ */
16
+ export declare function sliceCatalogEntriesByDescriptors(rawText: string, descriptors: CatalogEntryDescriptor[]): CatalogEntryChunk[];
17
+ /**
18
+ * Generic entry boundary detector for encyclopedias where entries are delineated by headers or page markers.
19
+ */
20
+ export declare function detectCatalogEntriesRegex(rawText: string, entryHeaderRegex: RegExp, options?: {
21
+ minLength?: number;
22
+ }): CatalogEntryChunk[];
@@ -0,0 +1,88 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.slugify = slugify;
4
+ exports.sliceCatalogEntriesByDescriptors = sliceCatalogEntriesByDescriptors;
5
+ exports.detectCatalogEntriesRegex = detectCatalogEntriesRegex;
6
+ /**
7
+ * Normalizes text into a clean lowercase URL/filesystem slug.
8
+ */
9
+ function slugify(text) {
10
+ return text
11
+ .normalize('NFD')
12
+ .replace(/[\u0300-\u036f]/g, '')
13
+ .toLowerCase()
14
+ .trim()
15
+ .replace(/\s+/g, '-')
16
+ .replace(/[^a-z0-9-]/g, '')
17
+ .replace(/--+/g, '-')
18
+ .replace(/^-+/, '')
19
+ .replace(/-+$/, '')
20
+ .slice(0, 80);
21
+ }
22
+ /**
23
+ * Slices raw monograph text into ordered catalog entry chunks based on explicit markers or titles.
24
+ */
25
+ function sliceCatalogEntriesByDescriptors(rawText, descriptors) {
26
+ const chunks = [];
27
+ // Locate position of each descriptor's start marker
28
+ const positions = [];
29
+ for (const desc of descriptors) {
30
+ const idx = rawText.indexOf(desc.startMarker);
31
+ if (idx !== -1) {
32
+ positions.push({ desc, index: idx });
33
+ }
34
+ }
35
+ // Sort descriptors strictly by their sequential appearance in the text
36
+ positions.sort((a, b) => a.index - b.index);
37
+ for (const [i, current] of positions.entries()) {
38
+ const nextIndex = positions.at(i + 1)?.index ?? rawText.length;
39
+ const textSlice = rawText.substring(current.index, nextIndex).trim();
40
+ const sequence = i + 1;
41
+ const slug = `${String(sequence).padStart(3, '0')}-${slugify(current.desc.title)}`;
42
+ chunks.push({
43
+ sequence,
44
+ rawTitle: current.desc.title,
45
+ slug,
46
+ scientificName: current.desc.scientificName,
47
+ family: current.desc.family,
48
+ pageRange: current.desc.pageRange,
49
+ text: textSlice,
50
+ });
51
+ }
52
+ return chunks;
53
+ }
54
+ /**
55
+ * Generic entry boundary detector for encyclopedias where entries are delineated by headers or page markers.
56
+ */
57
+ function detectCatalogEntriesRegex(rawText, entryHeaderRegex, options = {}) {
58
+ const minLength = options.minLength || 500;
59
+ const chunks = [];
60
+ const matches = [];
61
+ let currentOffset = 0;
62
+ while (currentOffset < rawText.length) {
63
+ entryHeaderRegex.lastIndex = 0;
64
+ const slice = rawText.slice(currentOffset);
65
+ const match = entryHeaderRegex.exec(slice);
66
+ if (!match) {
67
+ break;
68
+ }
69
+ const title = match[1] || match[0];
70
+ matches.push({ title: title.trim(), index: currentOffset + match.index });
71
+ currentOffset += match.index + Math.max(1, match[0].length);
72
+ }
73
+ for (const [i, current] of matches.entries()) {
74
+ const nextIndex = matches.at(i + 1)?.index ?? rawText.length;
75
+ const textSlice = rawText.substring(current.index, nextIndex).trim();
76
+ if (textSlice.length >= minLength) {
77
+ const sequence = chunks.length + 1;
78
+ const slug = `${String(sequence).padStart(3, '0')}-${slugify(current.title)}`;
79
+ chunks.push({
80
+ sequence,
81
+ rawTitle: current.title,
82
+ slug,
83
+ text: textSlice,
84
+ });
85
+ }
86
+ }
87
+ return chunks;
88
+ }
@@ -0,0 +1,33 @@
1
+ import { CatalogEntryChunk, CatalogIngestionOptions, CatalogIngestionSummary } from './types';
2
+ export interface CatalogMonographInput {
3
+ bookTitle: string;
4
+ description: string;
5
+ category: string;
6
+ authors?: string[];
7
+ publisher?: string;
8
+ publicationYear?: string | number;
9
+ edition?: string;
10
+ language?: string;
11
+ originalLanguage?: string;
12
+ tags?: string[];
13
+ introChapters?: Array<{
14
+ index: number;
15
+ title: string;
16
+ slug: string;
17
+ text: string;
18
+ summary?: string;
19
+ }>;
20
+ entries: CatalogEntryChunk[];
21
+ annexChapters?: Array<{
22
+ index: number;
23
+ title: string;
24
+ slug: string;
25
+ text: string;
26
+ summary?: string;
27
+ }>;
28
+ }
29
+ /**
30
+ * Ingests a structured catalog or encyclopedic monograph into a coherent OKF v0.2 collection.
31
+ * Decoupled from specific model providers and composable with domain taxonomy profiles.
32
+ */
33
+ export declare function ingestCatalogMonograph(input: CatalogMonographInput, options: CatalogIngestionOptions): Promise<CatalogIngestionSummary>;