@luzzle/core 0.0.32

package/dist/src/database/tables/pieces_items.schema.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=pieces_items.schema.js.map

package/dist/src/database/tables/pieces_items.schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pieces_items.schema.js","sourceRoot":"","sources":["../../../../src/database/tables/pieces_items.schema.ts"],"names":[],"mappings":""}

package/dist/src/database/tables/pieces_manager.schema.d.ts

@@ -0,0 +1,13 @@
+import type { Insertable, Updateable, Selectable } from 'kysely';
+import { cuid, date_added, date_updated } from '../utils.js';
+interface PieceManagerTable {
+    id: cuid;
+    date_added: date_added;
+    date_updated?: date_updated;
+    name: string;
+    schema: string;
+}
+type PieceManagerSelect = Selectable<PieceManagerTable>;
+type PieceManagerInsert = Insertable<PieceManagerTable>;
+type PieceManagerUpdate = Updateable<PieceManagerTable>;
+export { type PieceManagerTable, type PieceManagerSelect, type PieceManagerInsert, type PieceManagerUpdate, };

package/dist/src/database/tables/pieces_manager.schema.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=pieces_manager.schema.js.map

package/dist/src/database/tables/pieces_manager.schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pieces_manager.schema.js","sourceRoot":"","sources":["../../../../src/database/tables/pieces_manager.schema.ts"],"names":[],"mappings":""}

package/dist/src/database/utils.d.ts

@@ -0,0 +1,8 @@
+import { ColumnType } from 'kysely';
+type cuid = string;
+type date_added = ColumnType<number, undefined, never>;
+type date_updated = ColumnType<number | null, undefined, number>;
+type SchemaDateStringToDatabaseNumber<T> = {
+    [K in keyof T]: K extends `date_${string}` ? number : T[K] extends number ? string : T[K];
+};
+export { type cuid, type date_updated, type date_added, type SchemaDateStringToDatabaseNumber };

package/dist/src/database/utils.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=utils.js.map

package/dist/src/database/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../src/database/utils.ts"],"names":[],"mappings":""}

package/dist/src/database/utils.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/database/utils.test.js

@@ -0,0 +1,9 @@
+import { describe, expect, test } from 'vitest';
+import * as index from './utils.js';
+// https://github.com/vitest-dev/vitest/issues/3605
+describe('src/database/util.ts', () => {
+    test('schema', () => {
+        expect(index).toBeDefined();
+    });
+});
+//# sourceMappingURL=utils.test.js.map

package/dist/src/database/utils.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.test.js","sourceRoot":"","sources":["../../../src/database/utils.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAA;AAC/C,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AAEnC,mDAAmD;AAEnD,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACrC,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE;QACnB,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAA;IAC5B,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA"}

package/dist/src/index.d.ts

@@ -0,0 +1,11 @@
+import migrate from './database/migrations.js';
+import compile from './lib/ajv.js';
+import { sql } from 'kysely';
+import { extractFullMarkdown } from './lib/markdown.js';
+import { addFrontMatter } from './lib/frontmatter.js';
+import { getDatabaseClient } from './database/client.js';
+export * from './database/tables/index.js';
+export * from './pieces/index.js';
+export * from './llm/google.js';
+export * from './storage/index.js';
+export { getDatabaseClient, migrate, sql, compile, extractFullMarkdown, addFrontMatter };

package/dist/src/index.js

@@ -0,0 +1,12 @@
+import migrate from './database/migrations.js';
+import compile from './lib/ajv.js';
+import { sql } from 'kysely';
+import { extractFullMarkdown } from './lib/markdown.js';
+import { addFrontMatter } from './lib/frontmatter.js';
+import { getDatabaseClient } from './database/client.js';
+export * from './database/tables/index.js';
+export * from './pieces/index.js';
+export * from './llm/google.js';
+export * from './storage/index.js';
+export { getDatabaseClient, migrate, sql, compile, extractFullMarkdown, addFrontMatter };
+//# sourceMappingURL=index.js.map

package/dist/src/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,0BAA0B,CAAA;AAC9C,OAAO,OAAO,MAAM,cAAc,CAAA;AAClC,OAAO,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAA;AAC5B,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAA;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AAExD,cAAc,4BAA4B,CAAA;AAC1C,cAAc,mBAAmB,CAAA;AACjC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,oBAAoB,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,mBAAmB,EAAE,cAAc,EAAE,CAAA"}

package/dist/src/index.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/index.test.js

@@ -0,0 +1,9 @@
+import { describe, expect, test } from 'vitest';
+import * as index from './index.js';
+// https://github.com/vitest-dev/vitest/issues/3605
+describe('index.ts', () => {
+    test('schema', () => {
+        expect(index).toBeDefined();
+    });
+});
+//# sourceMappingURL=index.test.js.map

package/dist/src/index.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.test.js","sourceRoot":"","sources":["../../src/index.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAA;AAC/C,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AAEnC,mDAAmD;AAEnD,QAAQ,CAAC,UAAU,EAAE,GAAG,EAAE;IACzB,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE;QACnB,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAA;IAC5B,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA"}

package/dist/src/lib/ajv.d.ts

@@ -0,0 +1,6 @@
+import { ValidateFunction, JSONSchemaType } from 'ajv';
+export declare function commaSeparatedFormatValidator(data: string): boolean;
+export declare function dateFormatValidator(data: string): boolean;
+export declare function assetFormatValidator(data: string): boolean;
+export declare function paragraphValidator(data: string): boolean;
+export default function <T>(schema: JSONSchemaType<T>): ValidateFunction<T>;

package/dist/src/lib/ajv.js

@@ -0,0 +1,25 @@
+import Ajv from 'ajv';
+export function commaSeparatedFormatValidator(data) {
+    return typeof data === 'string';
+}
+export function dateFormatValidator(data) {
+    return !isNaN(Date.parse(data));
+}
+export function assetFormatValidator(data) {
+    return /^\.assets\/.+\/.+/.test(data);
+}
+export function paragraphValidator(data) {
+    return typeof data === 'string';
+}
+export default function (schema) {
+    const ajv = new Ajv();
+    ajv.addFormat('date', { type: 'string', validate: dateFormatValidator });
+    ajv.addFormat('asset', { type: 'string', validate: assetFormatValidator });
+    ajv.addFormat('comma-separated', {
+        type: 'string',
+        validate: commaSeparatedFormatValidator,
+    });
+    ajv.addFormat('paragraph', { type: 'string', validate: paragraphValidator });
+    return ajv.compile(schema);
+}
+//# sourceMappingURL=ajv.js.map

package/dist/src/lib/ajv.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ajv.js","sourceRoot":"","sources":["../../../src/lib/ajv.ts"],"names":[],"mappings":"AAAA,OAAO,GAAyC,MAAM,KAAK,CAAA;AAE3D,MAAM,UAAU,6BAA6B,CAAC,IAAY;IACzD,OAAO,OAAO,IAAI,KAAK,QAAQ,CAAA;AAChC,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC/C,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;AAChC,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,IAAY;IAChD,OAAO,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACtC,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC9C,OAAO,OAAO,IAAI,KAAK,QAAQ,CAAA;AAChC,CAAC;AAED,MAAM,CAAC,OAAO,WAAc,MAAyB;IACpD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAE,CAAA;IAErB,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,mBAAmB,EAAE,CAAC,CAAA;IACxE,GAAG,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,oBAAoB,EAAE,CAAC,CAAA;IAC1E,GAAG,CAAC,SAAS,CAAC,iBAAiB,EAAE;QAChC,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,6BAA6B;KACvC,CAAC,CAAA;IACF,GAAG,CAAC,SAAS,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,kBAAkB,EAAE,CAAC,CAAA;IAE5E,OAAO,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;AAC3B,CAAC"}

package/dist/src/lib/ajv.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/lib/ajv.test.js

@@ -0,0 +1,43 @@
+import { describe, expect, test, vi, afterEach } from 'vitest';
+import { makeSchema } from '../pieces/utils/piece.fixtures.js';
+import Ajv from 'ajv';
+import ajv, { assetFormatValidator, dateFormatValidator, commaSeparatedFormatValidator, paragraphValidator } from './ajv.js';
+vi.mock('ajv');
+const mocks = {
+    Ajv: vi.mocked(Ajv),
+    compile: vi.mocked(Ajv.prototype.compile),
+    addFormat: vi.mocked(Ajv.prototype.addFormat),
+};
+const spies = {};
+describe('src/lib/ajv.ts', () => {
+    afterEach(() => {
+        Object.values(mocks).forEach((mock) => {
+            mock.mockReset();
+        });
+        Object.keys(spies).forEach((key) => {
+            spies[key].mockRestore();
+            delete spies[key];
+        });
+    });
+    test('ajv', () => {
+        const schema = makeSchema();
+        ajv(schema);
+        expect(mocks.Ajv).toHaveBeenCalledTimes(1);
+        expect(mocks.compile).toHaveBeenCalledWith(schema);
+    });
+    test('commaSeparatedFormatValidator', () => {
+        expect(commaSeparatedFormatValidator('a,b,c')).toBe(true);
+    });
+    test('dateFormatValidator', () => {
+        expect(dateFormatValidator('2020-01-01')).toBe(true);
+        expect(dateFormatValidator('2020-01-32')).toBe(false);
+    });
+    test('assetFormatValidtor', () => {
+        expect(assetFormatValidator('.assets/1/2/3')).toBe(true);
+        expect(assetFormatValidator('./home/to/nowhere/5.jpg')).toBe(false);
+    });
+    test('paragraphFormatValidtor', () => {
+        expect(paragraphValidator('hi there')).toBe(true);
+    });
+});
+//# sourceMappingURL=ajv.test.js.map

package/dist/src/lib/ajv.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ajv.test.js","sourceRoot":"","sources":["../../../src/lib/ajv.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAgB,MAAM,QAAQ,CAAA;AAC5E,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAA;AAC9D,OAAO,GAAG,MAAM,KAAK,CAAA;AACrB,OAAO,GAAG,EAAE,EACX,oBAAoB,EACpB,mBAAmB,EACnB,6BAA6B,EAC7B,kBAAkB,EAClB,MAAM,UAAU,CAAA;AAEjB,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;AAEd,MAAM,KAAK,GAAG;IACb,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC;IACnB,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC;IACzC,SAAS,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC;CAC7C,CAAA;AAED,MAAM,KAAK,GAAoC,EAAE,CAAA;AAEjD,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC/B,SAAS,CAAC,GAAG,EAAE;QACd,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACrC,IAAI,CAAC,SAAS,EAAE,CAAA;QACjB,CAAC,CAAC,CAAA;QAEF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YAClC,KAAK,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAA;YACxB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAA;QAClB,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE;QAChB,MAAM,MAAM,GAAG,UAAU,EAAE,CAAA;QAE3B,GAAG,CAAC,MAAM,CAAC,CAAA;QAEX,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAA;QAC1C,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAA;IACnD,CAAC,CAAC,CAAA;IAEF,IAAI,CAAC,+BAA+B,EAAE,GAAG,EAAE;QAC1C,MAAM,CAAC,6BAA6B,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC1D,CAAC,CAAC,CAAA;IAEF,IAAI,CAAC,qBAAqB,EAAE,GAAG,EAAE;QAChC,MAAM,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpD,MAAM,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACtD,CAAC,CAAC,CAAA;IAEF,IAAI,CAAC,qBAAqB,EAAE,GAAG,EAAE;QAChC,MAAM,CAAC,oBAAoB,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxD,MAAM,CAAC,oBAAoB,CAAC,yBAAyB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACpE,CAAC,CAAC,CAAA;IAEF,IAAI,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACpC,MAAM,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAClD,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA"}

package/dist/src/lib/frontmatter.d.ts

@@ -0,0 +1,4 @@
+declare function addFrontMatter(markdown?: string | null, metadata?: {
+    [key: string]: unknown;
+}): string;
+export { addFrontMatter };

package/dist/src/lib/frontmatter.js

@@ -0,0 +1,8 @@
+import { stringify } from 'yaml';
+function addFrontMatter(markdown, metadata = {}) {
+    const yamlString = stringify(metadata);
+    const content = markdown?.trim() || '';
+    return `---\n${yamlString}---\n${content}\n`;
+}
+export { addFrontMatter };
+//# sourceMappingURL=frontmatter.js.map

package/dist/src/lib/frontmatter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"frontmatter.js","sourceRoot":"","sources":["../../../src/lib/frontmatter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,MAAM,CAAA;AAEhC,SAAS,cAAc,CACtB,QAAwB,EACxB,WAAuC,EAAE;IAEzC,MAAM,UAAU,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAA;IACtC,MAAM,OAAO,GAAG,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;IAEtC,OAAO,QAAQ,UAAU,QAAQ,OAAO,IAAI,CAAA;AAC7C,CAAC;AAED,OAAO,EAAE,cAAc,EAAE,CAAA"}

package/dist/src/lib/frontmatter.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/lib/frontmatter.test.js

@@ -0,0 +1,37 @@
+import { describe, expect, test, vi, afterEach } from 'vitest';
+import * as frontmatter from './frontmatter.js';
+import { stringify } from 'yaml';
+vi.mock('yaml');
+const mocks = {
+    stringify: vi.mocked(stringify),
+};
+describe('lib/frontmatter.ts', () => {
+    afterEach(() => {
+        Object.values(mocks).forEach((mock) => {
+            mock.mockReset();
+            mock.mockClear();
+        });
+    });
+    test('addFrontMatter', () => {
+        const markdown = 'this is markdown';
+        const metadata = {
+            a: 'awesome',
+            b: 'banana',
+        };
+        const yamlstring = 'yaml-front-matter';
+        mocks.stringify.mockReturnValueOnce(yamlstring);
+        const combined = frontmatter.addFrontMatter(markdown, metadata);
+        expect(combined).toBe(`---\n${yamlstring}---\n${markdown}\n`);
+    });
+    test('addFrontMatter with no markdown', () => {
+        const metadata = {
+            a: 'awesome',
+            b: 'banana',
+        };
+        const yamlstring = 'yaml-front-matter';
+        mocks.stringify.mockReturnValueOnce(yamlstring);
+        const combined = frontmatter.addFrontMatter(null, metadata);
+        expect(combined).toBe(`---\n${yamlstring}---\n\n`);
+    });
+});
+//# sourceMappingURL=frontmatter.test.js.map

package/dist/src/lib/frontmatter.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"frontmatter.test.js","sourceRoot":"","sources":["../../../src/lib/frontmatter.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AAC9D,OAAO,KAAK,WAAW,MAAM,kBAAkB,CAAA;AAC/C,OAAO,EAAE,SAAS,EAAE,MAAM,MAAM,CAAA;AAEhC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;AAEf,MAAM,KAAK,GAAG;IACb,SAAS,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC;CAC/B,CAAA;AAED,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;IACnC,SAAS,CAAC,GAAG,EAAE;QACd,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACrC,IAAI,CAAC,SAAS,EAAE,CAAA;YAChB,IAAI,CAAC,SAAS,EAAE,CAAA;QACjB,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,IAAI,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC3B,MAAM,QAAQ,GAAG,kBAAkB,CAAA;QACnC,MAAM,QAAQ,GAAG;YAChB,CAAC,EAAE,SAAS;YACZ,CAAC,EAAE,QAAQ;SACX,CAAA;QACD,MAAM,UAAU,GAAG,mBAAmB,CAAA;QAEtC,KAAK,CAAC,SAAS,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAA;QAE/C,MAAM,QAAQ,GAAG,WAAW,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;QAC/D,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,QAAQ,UAAU,QAAQ,QAAQ,IAAI,CAAC,CAAA;IAC9D,CAAC,CAAC,CAAA;IAEF,IAAI,CAAC,iCAAiC,EAAE,GAAG,EAAE;QAC5C,MAAM,QAAQ,GAAG;YAChB,CAAC,EAAE,SAAS;YACZ,CAAC,EAAE,QAAQ;SACX,CAAA;QACD,MAAM,UAAU,GAAG,mBAAmB,CAAA;QAEtC,KAAK,CAAC,SAAS,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAA;QAE/C,MAAM,QAAQ,GAAG,WAAW,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;QAC3D,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,QAAQ,UAAU,SAAS,CAAC,CAAA;IACnD,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA"}

package/dist/src/lib/markdown.d.ts

@@ -0,0 +1,7 @@
+declare function extractFullMarkdown(contents: string | Buffer): Promise<{
+    frontmatter: {
+        [key: string]: unknown;
+    };
+    markdown: string;
+}>;
+export { extractFullMarkdown };

package/dist/src/lib/markdown.js

@@ -0,0 +1,21 @@
+import { remark } from 'remark';
+import remarkFrontMatter from 'remark-frontmatter';
+import { visit } from 'unist-util-visit';
+import YAML from 'yaml';
+async function extractFullMarkdown(contents) {
+    let frontmatter = {};
+    let markdown = contents.toString();
+    await remark()
+        .use(remarkFrontMatter)
+        .use(() => (tree) => {
+        visit(tree, 'yaml', function (node) {
+            frontmatter = YAML.parse(node.value);
+            const endPosition = node.position?.end.offset;
+            markdown = markdown.slice(endPosition).trim();
+        });
+    })
+        .process(contents);
+    return { frontmatter, markdown };
+}
+export { extractFullMarkdown };
+//# sourceMappingURL=markdown.js.map

package/dist/src/lib/markdown.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../../src/lib/markdown.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AAC/B,OAAO,iBAAiB,MAAM,oBAAoB,CAAA;AAClD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAA;AACxC,OAAO,IAAI,MAAM,MAAM,CAAA;AAEvB,KAAK,UAAU,mBAAmB,CACjC,QAAyB;IAEzB,IAAI,WAAW,GAAG,EAAE,CAAA;IACpB,IAAI,QAAQ,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAA;IAElC,MAAM,MAAM,EAAE;SACZ,GAAG,CAAC,iBAAiB,CAAC;SACtB,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,IAAU,EAAE,EAAE;QACzB,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,IAAI;YACjC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACpC,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAA;YAC7C,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,CAAA;QAC9C,CAAC,CAAC,CAAA;IACH,CAAC,CAAC;SACD,OAAO,CAAC,QAAQ,CAAC,CAAA;IAEnB,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAA;AACjC,CAAC;AAED,OAAO,EAAE,mBAAmB,EAAE,CAAA"}

package/dist/src/lib/markdown.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/lib/markdown.test.js

@@ -0,0 +1,34 @@
+import { describe, expect, test, afterEach } from 'vitest';
+import * as markdown from './markdown.js';
+const mocks = {};
+describe('./lib/markdown.ts', () => {
+    afterEach(() => {
+        Object.values(mocks).forEach(() => {
+            //mock.mockReset()
+            //mock.mockClear()
+        });
+    });
+    test('extract markdown and frontmatter', async () => {
+        const contents = '---\na: awesome\nb: banana\n---\nthis is markdown';
+        const extracted = await markdown.extractFullMarkdown(contents);
+        expect(extracted.frontmatter).toEqual({ a: 'awesome', b: 'banana' });
+        expect(extracted.markdown).toBe('this is markdown');
+    });
+    test('should not escape footnotes', async () => {
+        const contents = `--- 
+a: awesome
+b: banana
+---
+Here is a footnote reference,[^1] and another.[^longnote]
+
+[^1]: Here is the footnote.
+[^longnote]: Here's one with multiple blocks.
+`;
+        const extracted = await markdown.extractFullMarkdown(contents);
+        expect(extracted.markdown).toBe(`Here is a footnote reference,[^1] and another.[^longnote]
+
+[^1]: Here is the footnote.
+[^longnote]: Here's one with multiple blocks.`);
+    });
+});
+//# sourceMappingURL=markdown.test.js.map

package/dist/src/lib/markdown.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown.test.js","sourceRoot":"","sources":["../../../src/lib/markdown.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AAC1D,OAAO,KAAK,QAAQ,MAAM,eAAe,CAAA;AAEzC,MAAM,KAAK,GAAG,EAAE,CAAA;AAEhB,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IAClC,SAAS,CAAC,GAAG,EAAE;QACd,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE;YACjC,kBAAkB;YAClB,kBAAkB;QACnB,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEH,IAAI,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,QAAQ,GAAG,mDAAmD,CAAA;QAEpE,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAA;QAE9D,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAA;QACpE,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAA;IACpD,CAAC,CAAC,CAAA;IAEF,IAAI,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;QAC9C,MAAM,QAAQ,GAAG;;;;;;;;CAQlB,CAAA;QACC,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAA;QAC9D,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,IAAI,CAC9B;;;8CAG2C,CAC3C,CAAA;IACF,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA"}

package/dist/src/llm/google.d.ts

@@ -0,0 +1,6 @@
+import { JSONSchemaType } from 'ajv';
+import { type PieceFrontmatter } from '../pieces/index.js';
+declare function pieceFrontMatterFromPrompt(apiKey: string, schema: JSONSchemaType<PieceFrontmatter>, prompt: string, files?: Array<string | Buffer>): Promise<{
+    [k: string]: string | number | boolean | number[] | string[] | boolean[];
+}>;
+export { pieceFrontMatterFromPrompt };

package/dist/src/llm/google.js

@@ -0,0 +1,91 @@
+import { fileTypeFromBuffer, fileTypeFromFile } from 'file-type';
+import { createPartFromUri, GoogleGenAI, HarmCategory, HarmBlockThreshold, } from '@google/genai';
+import { readFile } from 'fs/promises';
+import path from 'path';
+const MODEL_NAME = 'gemini-2.5-flash';
+const safetySettings = [
+    {
+        category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+        threshold: HarmBlockThreshold.BLOCK_NONE,
+    },
+    {
+        category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+        threshold: HarmBlockThreshold.BLOCK_NONE,
+    },
+    {
+        category: HarmCategory.HARM_CATEGORY_CIVIC_INTEGRITY,
+        threshold: HarmBlockThreshold.BLOCK_NONE,
+    },
+];
+function getClient(apiKey) {
+    return new GoogleGenAI({ apiKey });
+}
+async function extractPartFromFile(file, genAI) {
+    const isBuffer = Buffer.isBuffer(file);
+    const fileType = isBuffer ? await fileTypeFromBuffer(file) : await fileTypeFromFile(file);
+    const mimeType = fileType?.mime;
+    // assume it's non binary
+    if (!mimeType) {
+        const fileContent = isBuffer ? file.toString() : await readFile(file, 'utf-8');
+        const fileName = isBuffer ? 'string buffer of downloaded file' : path.basename(file);
+        const data = [
+            `---[start] embedding ${fileName}---`,
+            fileContent,
+            `---[end] embedding of ${fileName}---`,
+        ];
+        return data.join('\n');
+    }
+    else {
+        const uploadedFile = await genAI.files.upload({
+            file: isBuffer ? new Blob([file]) : file,
+            config: {
+                mimeType,
+            },
+        });
+        // Wait for the file to be processed.
+        let getFile = await genAI.files.get({ name: uploadedFile.name });
+        while (getFile.state === 'PROCESSING') {
+            getFile = await genAI.files.get({ name: uploadedFile.name });
+            await new Promise((resolve) => {
+                setTimeout(resolve, 5000);
+            });
+        }
+        if (uploadedFile.state === 'FAILED') {
+            throw new Error('File processing failed.');
+        }
+        if (uploadedFile.uri && uploadedFile.mimeType) {
+            const fileContent = createPartFromUri(uploadedFile.uri, uploadedFile.mimeType);
+            return fileContent;
+        }
+    }
+    throw new Error('File processing failed.');
+}
+async function pieceFrontMatterFromPrompt(apiKey, schema, prompt, files) {
+    const genAI = getClient(apiKey);
+    const content = [prompt];
+    if (files) {
+        for (const file of files) {
+            const part = await extractPartFromFile(file, genAI);
+            content.push(part);
+        }
+    }
+    const result = await genAI.models.generateContent({
+        model: MODEL_NAME,
+        contents: content,
+        config: {
+            responseMimeType: 'application/json',
+            safetySettings: safetySettings,
+            responseJsonSchema: schema,
+            systemInstruction: `you are an assistant that helps generate JSON metadata for a record that will be added to a collection of similar records. 
+
+if you are provided pdf attachments, images or other text based files, please prioritize them as inputs for generating metadata for the record. 
+
+you are also given a responseJsonSchema to guide your output. each field in the schema has a description and examples to help guide what the intention of each field is and what values to expect.`,
+        },
+    });
+    const metadata = result.text || '{}';
+    const frontmatter = JSON.parse(metadata);
+    return Object.fromEntries(Object.entries(frontmatter).filter(([, value]) => value !== null || value === ''));
+}
+export { pieceFrontMatterFromPrompt };
+//# sourceMappingURL=google.js.map

package/dist/src/llm/google.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"google.js","sourceRoot":"","sources":["../../../src/llm/google.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAA;AAChE,OAAO,EAEN,iBAAiB,EACjB,WAAW,EACX,YAAY,EACZ,kBAAkB,GAElB,MAAM,eAAe,CAAA;AACtB,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,IAAI,MAAM,MAAM,CAAA;AAIvB,MAAM,UAAU,GAAG,kBAAkB,CAAA;AAErC,MAAM,cAAc,GAAoB;IACvC;QACC,QAAQ,EAAE,YAAY,CAAC,+BAA+B;QACtD,SAAS,EAAE,kBAAkB,CAAC,UAAU;KACxC;IACD;QACC,QAAQ,EAAE,YAAY,CAAC,+BAA+B;QACtD,SAAS,EAAE,kBAAkB,CAAC,UAAU;KACxC;IACD;QACC,QAAQ,EAAE,YAAY,CAAC,6BAA6B;QACpD,SAAS,EAAE,kBAAkB,CAAC,UAAU;KACxC;CACD,CAAA;AAED,SAAS,SAAS,CAAC,MAAc;IAChC,OAAO,IAAI,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAA;AACnC,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,IAAqB,EAAE,KAAkB;IAC3E,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;IACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,gBAAgB,CAAC,IAAI,CAAC,CAAA;IACzF,MAAM,QAAQ,GAAG,QAAQ,EAAE,IAAI,CAAA;IAE/B,yBAAyB;IACzB,IAAI,CAAC,QAAQ,EAAE,CAAC;QACf,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;QAC9E,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,kCAAkC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QACpF,MAAM,IAAI,GAAG;YACZ,wBAAwB,QAAQ,KAAK;YACrC,WAAW;YACX,yBAAyB,QAAQ,KAAK;SACtC,CAAA;QAED,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACvB,CAAC;SAAM,CAAC;QACP,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;YAC7C,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;YACxC,MAAM,EAAE;gBACP,QAAQ;aACR;SACD,CAAC,CAAA;QAEF,qCAAqC;QACrC,IAAI,OAAO,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,YAAY,CAAC,IAAc,EAAE,CAAC,CAAA;QAE1E,OAAO,OAAO,CAAC,KAAK,KAAK,YAAY,EAAE,CAAC;YACvC,OAAO,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,YAAY,CAAC,IAAc,EAAE,CAAC,CAAA;YAEtE,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC7B,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;YAC1B,CAAC,CAAC,CAAA;QACH,CAAC;QAED,IAAI,YAAY,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAA;QAC3C,CAAC;QAED,IAAI,YAAY,CAAC,GAAG,IAAI,YAAY,CAAC,QAAQ,EAAE,CAAC;YAC/C,MAAM,WAAW,GAAG,iBAAiB,CAAC,YAAY,CAAC,GAAG,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAA;YAC9E,OAAO,WAAW,CAAA;QACnB,CAAC;IACF,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAA;AAC3C,CAAC;AAED,KAAK,UAAU,0BAA0B,CACxC,MAAc,EACd,MAAwC,EACxC,MAAc,EACd,KAA8B;IAE9B,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAA;IAE/B,MAAM,OAAO,GAAqB,CAAC,MAAM,CAAC,CAAA;IAE1C,IAAI,KAAK,EAAE,CAAC;QACX,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;YACnD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACnB,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,eAAe,CAAC;QACjD,KAAK,EAAE,UAAU;QACjB,QAAQ,EAAE,OAAO;QACjB,MAAM,EAAE;YACP,gBAAgB,EAAE,kBAAkB;YACpC,cAAc,EAAE,cAAc;YAC9B,kBAAkB,EAAE,MAAM;YAC1B,iBAAiB,EAAE;;;;mMAI6K;SAChM;KACD,CAAC,CAAA;IAEF,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,IAAI,IAAI,CAAA;IACpC,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAqB,CAAA;IAE5D,OAAO,MAAM,CAAC,WAAW,CACxB,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC,CACjF,CAAA;AACF,CAAC;AAED,OAAO,EAAE,0BAA0B,EAAE,CAAA"}

package/dist/src/llm/google.test.d.ts

@@ -0,0 +1 @@
+export {};