@luzzle/core 0.0.62 → 0.0.64

package/README.md

@@ -1,14 +1,44 @@
 # @luzzle/core 🫀
 
-The foundational implementation of the Luzzle specification. It solves the inherent challenges of managing large-scale digital records using plain text.
+TypeScript implementation of the Luzzle specification.
 
-## Challenges Solved 🏗️
+Luzzle is a specification designed to solve the challenges of managing digital
+records using plain text files instead of proprietary databases or spreadsheets.
+`@luzzle/core` provides the tooling to define, parse, and validate these text
+records ("pieces") and attachments so they can be easily queried, indexed, and
+synced to a database.
 
-* **Consistency**: Enforces structure across thousands of records using JSON schemas.
-* **Organization**: Standardizes where files and attachments (`.assets/`) live within an archive.
-* **Scale**: Maps thousands of text files into a high-performance SQLite index, enabling complex queries and fast searching without slow filesystem traversal.
-* **Portability**: Provides a unified Storage abstraction so your data remains accessible anywhere.
+---
 
-## How it works 🚂
+## Core Concepts 🏗️
 
-Core treats your filesystem as the primary database. It parses Markdown, validates frontmatter, and maintains a derivative SQLite cache for application-layer performance.
+### 1. Pieces & Schemas
+
+The core of the Luzzle spec is the **Piece**: a standard Markdown file with
+structured YAML frontmatter. `@luzzle/core` validates these pieces using
+standard JSON schemas (typically defined in your archive under
+`.luzzle/schemas/`). This gives you uniform, type-safe records without holding
+your data hostage.
+
+### 2. Assets & Attachments
+
+A digital garden is not just text. `@luzzle/core` defines conventions for how
+non-text media (like images or PDF attachments) are stored within the
+`.assets/` directory and referenced in a piece's metadata.
+
+### 3. File Naming Convention
+
+To associate a piece with its schema, Luzzle relies on a specific file naming
+convention: `name.[piece-type].md` (e.g., `dune.books.md`). This naming
+convention solves the problem of quickly identifying a piece's type to map it
+to the correct validator. _Note: This convention might be evolved or removed
+in the future, but remains the current way piece types are discovered._
+
+### 4. Simple Filesystem Backend
+
+To adhere to the Unix philosophy, `@luzzle/core` **only** implements a simple
+local filesystem backend. It reads Markdown files, validates frontmatter, and
+updates a derivative SQLite index for query performance.
+
+Upstream apps (such as `@luzzle/web` or custom deploy scripts) are responsible
+for handling remote storage backends (like WebDAV or cloud storage syncing).

package/dist/src/index.d.ts

@@ -4,9 +4,9 @@ import { sql } from 'kysely';
 import { extractFullMarkdown } from './lib/markdown.js';
 import { addFrontMatter } from './lib/frontmatter.js';
 import { getDatabaseClient } from './database/client.js';
-import { filterFrontmatterFields, findFrontmatterField, getFrontmatterValue, getFrontmatterValues, resolveFieldPaths, setFrontmatterValue, unsetFrontmatterValue } from './pieces/index.js';
+import { filterFrontmatterFields, findFrontmatterField, getFrontmatterValue, getFrontmatterValues, resolveFieldPaths, setFrontmatterValue, unsetFrontmatterValue, savePieceAsset } from './pieces/index.js';
 export * from './database/tables/index.js';
 export * from './pieces/index.js';
 export * from './llm/google.js';
 export * from './storage/index.js';
-export { getDatabaseClient, migrate, sql, compile, extractFullMarkdown, addFrontMatter, filterFrontmatterFields, findFrontmatterField, getFrontmatterValue, getFrontmatterValues, resolveFieldPaths, setFrontmatterValue, unsetFrontmatterValue };
+export { getDatabaseClient, migrate, sql, compile, extractFullMarkdown, addFrontMatter, filterFrontmatterFields, findFrontmatterField, getFrontmatterValue, getFrontmatterValues, resolveFieldPaths, setFrontmatterValue, unsetFrontmatterValue, savePieceAsset, };

package/dist/src/index.js

@@ -4,10 +4,10 @@ import { sql } from 'kysely';
 import { extractFullMarkdown } from './lib/markdown.js';
 import { addFrontMatter } from './lib/frontmatter.js';
 import { getDatabaseClient } from './database/client.js';
-import { filterFrontmatterFields, findFrontmatterField, getFrontmatterValue, getFrontmatterValues, resolveFieldPaths, setFrontmatterValue, unsetFrontmatterValue } from './pieces/index.js';
+import { filterFrontmatterFields, findFrontmatterField, getFrontmatterValue, getFrontmatterValues, resolveFieldPaths, setFrontmatterValue, unsetFrontmatterValue, savePieceAsset, } from './pieces/index.js';
 export * from './database/tables/index.js';
 export * from './pieces/index.js';
 export * from './llm/google.js';
 export * from './storage/index.js';
-export { getDatabaseClient, migrate, sql, compile, extractFullMarkdown, addFrontMatter, filterFrontmatterFields, findFrontmatterField, getFrontmatterValue, getFrontmatterValues, resolveFieldPaths, setFrontmatterValue, unsetFrontmatterValue };
+export { getDatabaseClient, migrate, sql, compile, extractFullMarkdown, addFrontMatter, filterFrontmatterFields, findFrontmatterField, getFrontmatterValue, getFrontmatterValues, resolveFieldPaths, setFrontmatterValue, unsetFrontmatterValue, savePieceAsset, };
 //# sourceMappingURL=index.js.map

package/dist/src/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,0BAA0B,CAAA;AAC9C,OAAO,OAAO,MAAM,cAAc,CAAA;AAClC,OAAO,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAA;AAC5B,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAA;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AACxD,OAAO,EAAE,uBAAuB,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAA;AAE3L,cAAc,4BAA4B,CAAA;AAC1C,cAAc,mBAAmB,CAAA;AACjC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,oBAAoB,CAAA;AAElC,OAAO,EACN,iBAAiB,EACjB,OAAO,EACP,GAAG,EACH,OAAO,EACP,mBAAmB,EACnB,cAAc,EACd,uBAAuB,EACvB,oBAAoB,EACpB,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EACjB,mBAAmB,EACnB,qBAAqB,EACrB,CAAA","sourcesContent":["import migrate from './database/migrations.js'\nimport compile from './lib/ajv.js'\nimport { sql } from 'kysely'\nimport { extractFullMarkdown } from './lib/markdown.js'\nimport { addFrontMatter } from './lib/frontmatter.js'\nimport { getDatabaseClient } from './database/client.js'\nimport { filterFrontmatterFields, findFrontmatterField, getFrontmatterValue, getFrontmatterValues, resolveFieldPaths, setFrontmatterValue, unsetFrontmatterValue } from './pieces/index.js'\n\nexport * from './database/tables/index.js'\nexport * from './pieces/index.js'\nexport * from './llm/google.js'\nexport * from './storage/index.js'\n\nexport {\n\tgetDatabaseClient,\n\tmigrate,\n\tsql,\n\tcompile,\n\textractFullMarkdown,\n\taddFrontMatter,\n\tfilterFrontmatterFields,\n\tfindFrontmatterField,\n\tgetFrontmatterValue,\n\tgetFrontmatterValues,\n\tresolveFieldPaths,\n\tsetFrontmatterValue,\n\tunsetFrontmatterValue\n}\n"]}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,0BAA0B,CAAA;AAC9C,OAAO,OAAO,MAAM,cAAc,CAAA;AAClC,OAAO,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAA;AAC5B,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAA;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AACxD,OAAO,EACN,uBAAuB,EACvB,oBAAoB,EACpB,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EACjB,mBAAmB,EACnB,qBAAqB,EACrB,cAAc,GACd,MAAM,mBAAmB,CAAA;AAE1B,cAAc,4BAA4B,CAAA;AAC1C,cAAc,mBAAmB,CAAA;AACjC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,oBAAoB,CAAA;AAElC,OAAO,EACN,iBAAiB,EACjB,OAAO,EACP,GAAG,EACH,OAAO,EACP,mBAAmB,EACnB,cAAc,EACd,uBAAuB,EACvB,oBAAoB,EACpB,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EACjB,mBAAmB,EACnB,qBAAqB,EACrB,cAAc,GACd,CAAA","sourcesContent":["import migrate from './database/migrations.js'\nimport compile from './lib/ajv.js'\nimport { sql } from 'kysely'\nimport { extractFullMarkdown } from './lib/markdown.js'\nimport { addFrontMatter } from './lib/frontmatter.js'\nimport { getDatabaseClient } from './database/client.js'\nimport {\n\tfilterFrontmatterFields,\n\tfindFrontmatterField,\n\tgetFrontmatterValue,\n\tgetFrontmatterValues,\n\tresolveFieldPaths,\n\tsetFrontmatterValue,\n\tunsetFrontmatterValue,\n\tsavePieceAsset,\n} from './pieces/index.js'\n\nexport * from './database/tables/index.js'\nexport * from './pieces/index.js'\nexport * from './llm/google.js'\nexport * from './storage/index.js'\n\nexport {\n\tgetDatabaseClient,\n\tmigrate,\n\tsql,\n\tcompile,\n\textractFullMarkdown,\n\taddFrontMatter,\n\tfilterFrontmatterFields,\n\tfindFrontmatterField,\n\tgetFrontmatterValue,\n\tgetFrontmatterValues,\n\tresolveFieldPaths,\n\tsetFrontmatterValue,\n\tunsetFrontmatterValue,\n\tsavePieceAsset,\n}\n"]}

package/dist/src/pieces/Piece.js

@@ -6,7 +6,7 @@ import { Readable } from 'stream';
 import { databaseValueToPieceFrontmatterValue, getPieceFrontmatterSchemaFields, initializePieceFrontMatter, } from './utils/frontmatter.js';
 import { findFrontmatterField, setFrontmatterValue, unsetFrontmatterValue, getFrontmatterValue, } from './utils/frontmatter.path.js';
 import { makePieceMarkdown, makePieceMarkdownString } from './utils/markdown.js';
-import { isAttachableStream, calculateHashFromFile, makePieceAttachment, makePieceValue, } from './utils/piece.js';
+import { isAttachableStream, calculateHashFromFile, savePieceFieldAsset, makePieceValue, } from './utils/piece.js';
 import compile from '../lib/ajv.js';
 import { getValidatePieceItemErrors, makePieceItemInsertable, makePieceItemUpdatable, validatePieceItem, } from './item.js';
 import { extractFullMarkdown } from '../lib/markdown.js';
@@ -221,7 +221,7 @@ class Piece {
             for (const one of values) {
                 const pieceValue = await makePieceValue(itemField, one);
                 const val = isAttachableStream(pieceValue)
-                    ? await makePieceAttachment(markdown.filePath, itemField, pieceValue, this._storage)
+                    ? await savePieceFieldAsset(markdown.filePath, itemField, pieceValue, this._storage)
                     : pieceValue;
                 setFrontmatterValue(updatedFrontmatter, fieldPath, val);
             }

package/dist/src/pieces/Piece.js.map

@@ -1 +1 @@
-{"version":3,"file":"Piece.js","sourceRoot":"","sources":["../../../src/pieces/Piece.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AAC5D,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAA;AACzB,OAAO,IAAI,MAAM,MAAM,CAAA;AACvB,OAAO,OAAO,MAAM,uBAAuB,CAAA;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAA;AACjC,OAAO,EACN,oCAAoC,EACpC,+BAA+B,EAC/B,0BAA0B,GAK1B,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EACN,oBAAoB,EACpB,mBAAmB,EACnB,qBAAqB,EACrB,mBAAmB,GACnB,MAAM,6BAA6B,CAAA;AAEpC,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAiB,MAAM,qBAAqB,CAAA;AAC/F,OAAO,EACN,kBAAkB,EAClB,qBAAqB,EACrB,mBAAmB,EACnB,cAAc,GACd,MAAM,kBAAkB,CAAA;AAEzB,OAAO,OAAO,MAAM,eAAe,CAAA;AACnC,OAAO,EACN,0BAA0B,EAC1B,uBAAuB,EACvB,sBAAsB,EACtB,iBAAiB,GACjB,MAAM,WAAW,CAAA;AAClB,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AACxF,OAAO,EAAE,2BAA2B,EAAE,MAAM,aAAa,CAAA;AA8BzD,MAAM,KAAK;IACF,UAAU,CAAgC;IAC1C,OAAO,CAA2B;IAClC,QAAQ,CAAe;IACvB,UAAU,CAAQ;IAClB,OAAO,CAAqC;IAEpD,YAAY,SAAiB,EAAE,OAAsB,EAAE,MAAiC;QACvF,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAA;QACvB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAA;QAE3B,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,qCAAqC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;QACvF,CAAC;IACF,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,SAAiB,EAAE,IAAY;QAC3C,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;QAC1B,MAAM,QAAQ,GAAG,GAAG,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,2BAA2B,EAAE,CAAA;QACtE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;QAC3C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAE/C,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,EAAE,CAAC,CAAA;QAChD,CAAC;QAED,MAAM,WAAW,GAAG,0BAA0B,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAM,CAAA;QACvE,OAAO,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,SAAS,EAAE,WAAW,CAAC,CAAA;IACxE,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAY;QACxB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAE/C,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAChC,OAAM;QACP,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,iBAAiB,CAAC,CAAA;IAC1C,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,UAAU,CAAA;IACvB,CAAC;IAED,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,OAAO,CAAA;IACpB,CAAC;IAED,IAAc,SAAS;QACtB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,OAAO,CAAI,IAAI,CAAC,OAAO,CAAC,CAAA;QAC7D,OAAO,IAAI,CAAC,UAAU,CAAA;IACvB,CAAC;IAED,IAAI,MAAM;QACT,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,+BAA+B,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC5E,OAAO,IAAI,CAAC,OAAO,CAAA;IACpB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAY,EAAE,EAAkB;QAChD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAA;QAEjE,IAAI,QAAQ,EAAE,CAAC;YACd,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;YACtC,MAAM,UAAU,GAAG,KAAK,EAAE,YAAY,IAAI,KAAK,EAAE,UAAU,CAAA;YAE3D,OAAO,CAAC,UAAU,IAAI,QAAQ,CAAC,aAAa,CAAC,OAAO,EAAE,GAAG,UAAU,CAAA;QACpE,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,iBAAiB,CAAC,CAAA;IAC1C,CAAC;IAED,QAAQ,CAAC,QAA0B;QAClC,MAAM,KAAK,GAAG,iBAAiB,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAA;QAEzD,IAAI,KAAK,EAAE,CAAC;YACX,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;QACzB,CAAC;aAAM,CAAC;YACP,MAAM,MAAM,GAAG,0BAA0B,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACzD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAA;QAClC,CAAC;IACF,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAY;QACrB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAE/C,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;YAC3D,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAAC,QAAQ,CAAC,CAAA;YAEhD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvB,OAAO,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAgB,CAAC,CAAA;YACtF,CAAC;QACF,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,iBAAiB,CAAC,CAAA;IAC1C,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAA0B;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAEzC,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;YACvB,MAAM,cAAc,GAAG,uBAAuB,CAAC,QAAQ,CAAC,CAAA;YACxD,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAA;QACjE,CAAC;aAAM,CAAC;YACP,MAAM,IAAI,KAAK,CACd,mBAAmB,QAAQ,CAAC,QAAQ,gBAAgB,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACjF,CAAA;QACF,CAAC;IACF,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAkB,EAAE,KAAe,EAAE,OAA6B;QAC7E,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAA;QACjE,MAAM,aAAa,GAAG,IAAI,GAAG,CAAS,KAAK,CAAC,CAAA;QAC5C,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAA;QACrF,MAAM,YAAY,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;QAClE,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;QAE1C,OAAO,MAAM,CAAC,GAAG,CAChB,KAAK,EAAE,IAAY,EAA6B,EAAE;YACjD,IAAI,CAAC;gBACJ,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;oBACtB,MAAM,UAAU,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;gBAC3B,CAAC;gBACD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;YAClC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,wBAAwB,KAAK,EAAE,EAAE,CAAA;YACvE,CAAC;QACF,CAAC,EACD,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,MAAM,EAAE,CACgB,CAAA;IAChD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAkB,EAAE,KAAe,EAAE,OAA+C;QAC9F,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAEnC,OAAO,MAAM,CAAC,GAAG,CAChB,KAAK,EAAE,IAAY,EAA4B,EAAE;YAChD,IAAI,CAAC;gBACJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;gBACrC,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAA;gBAEvD,IAAI,OAAO,EAAE,CAAC;oBACb,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;oBACpE,MAAM,OAAO,GAAG,MAAM,qBAAqB,CAAC,UAAU,CAAC,CAAA;oBACvD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAA;oBACnD,IAAI,OAAO,EAAE,KAAK,IAAI,KAAK,EAAE,YAAY,KAAK,OAAO,EAAE,CAAC;wBACvD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;4BACtB,MAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAA;wBACrD,CAAC;wBACD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAA;oBACnC,CAAC;oBAED,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;wBACtB,MAAM,WAAW,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;oBAClD,CAAC;oBAED,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAA;gBACnC,CAAC;qBAAM,CAAC;oBACP,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;wBACtB,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAA;oBACzC,CAAC;oBACD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;gBACjC,CAAC;YACF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,wBAAwB,KAAK,EAAE,EAAE,CAAA;YACvE,CAAC;QACF,CAAC,EACD,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,MAAM,EAAE,CACe,CAAA;IAC/C,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,EAAkB,EAAE,QAA0B;QACnE,MAAM,WAAW,GAAG,uBAAuB,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;QACpF,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;QACpE,MAAM,IAAI,GAAG,MAAM,qBAAqB,CAAC,UAAU,CAAC,CAAA;QAEpD,MAAM,UAAU,CAAC,EAAE,EAAE,WAAW,CAAC,CAAA;QACjC,MAAM,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;IAC5C,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,EAAkB,EAAE,QAA0B;QAChE,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAEvD,IAAI,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAA;QACrD,CAAC;aAAM,CAAC;YACP,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAA;QACzC,CAAC;IACF,CAAC;IAED,KAAK,CAAC,kBAAkB,CACvB,EAAkB,EAClB,QAA0B,EAC1B,IAAsC;QAEtC,MAAM,WAAW,GAAG,sBAAsB,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAA;QAC/E,MAAM,UAAU,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;QACpD,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;QACpE,MAAM,IAAI,GAAG,MAAM,qBAAqB,CAAC,UAAU,CAAC,CAAA;QAEpD,MAAM,WAAW,CAAC,EAAE,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IAC5C,CAAC;IAED,UAAU,CAAC,IAAsC;QAChD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QACrD,MAAM,eAAe,GAA4B,EAAE,CAAA;QACnD,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;QACzC,MAAM,MAAM,GAAG,+BAA+B,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,CAClE,CAAC,CAAC,EAAuD,EAAE,CAC1D,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CACtC,CAAA;QAED,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;YACxB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAA;YACvB,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAA;YAE/B,eAAe,CAAC,IAAI,CAAC,GAAG,oCAAoC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;QAC3E,CAAC,CAAC,CAAA;QAEF,OAAO,iBAAiB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,aAAa,EAAE,eAAoB,CAAC,CAAA;IAC9F,CAAC;IAED,QAAQ,CAAC,QAA0B,EAAE,SAAiB;QACrD,OAAO,mBAAmB,CAAC,QAAQ,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;IAC5D,CAAC;IAED,KAAK,CAAC,SAAS,CACd,QAA0B,EAC1B,MAA+B;QAE/B,IAAI,eAAe,GAAG,QAAQ,CAAA;QAE9B,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;YAChC,eAAe,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,eAAe,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAA;QACrF,CAAC;QAED,OAAO,eAAe,CAAA;IACvB,CAAC;IAED,KAAK,CAAC,YAAY,CACjB,QAA0B,EAC1B,MAAgB;QAEhB,IAAI,eAAe,GAAG,QAAsD,CAAA;QAE5E,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC5B,eAAe,GAAG,CAAC,MAAM,IAAI,CAAC,WAAW,CACxC,eAA8C,EAC9C,KAAK,CACL,CAA+C,CAAA;QACjD,CAAC;QAED,OAAO,eAAe,CAAA;IACvB,CAAC;IAED,KAAK,CAAC,QAAQ,CACb,QAA0B,EAC1B,SAAiB,EACjB,KAAc;QAEd,MAAM,UAAU,GAAG,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;QAE/D,IAAI,CAAC,UAAU,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,sBAAsB,IAAI,CAAC,UAAU,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAA;QAC1F,CAAC;QAED,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,KAAK,OAAO,CAAA;QAC3C,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,UAAU,CAAA;QACvF,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;QAErD,MAAM,kBAAkB,GAAG,eAAe,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAA;QAEhE,IAAI,CAAC;YACJ,IAAI,OAAO,EAAE,CAAC;gBACb,MAAM,OAAO,GAAG,mBAAmB,CAAC,kBAAkB,EAAE,SAAS,CAAC,CAAA;gBAClE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC7B,mBAAmB,CAAC,kBAAkB,EAAE,SAAS,EAAE,EAAsC,CAAC,CAAA;gBAC3F,CAAC;YACF,CAAC;YAED,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;gBAC1B,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;gBAEvD,MAAM,GAAG,GAAG,kBAAkB,CAAC,UAAU,CAAC;oBACzC,CAAC,CAAC,MAAM,mBAAmB,CAC1B,QAAQ,CAAC,QAAQ,EACjB,SAAwC,EACxC,UAAU,EACV,IAAI,CAAC,QAAQ,CACb;oBACD,CAAC,CAAE,UAAoC,CAAA;gBAExC,mBAAmB,CAAC,kBAAkB,EAAE,SAAS,EAAE,GAAG,CAAC,CAAA;YACxD,CAAC;QACF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACZ,MAAM,KAAK,GAAG,CAAU,CAAA;YACxB,OAAO,CAAC,KAAK,CAAC,uBAAuB,SAAS,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAA;YACnE,OAAO,QAAQ,CAAA;QAChB,CAAC;QAED,OAAO,iBAAiB,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAA;IAC/F,CAAC;IAED,KAAK,CAAC,WAAW,CAChB,QAA0B,EAC1B,SAAiB,EACjB,KAAiC;QAEjC,MAAM,UAAU,GAAG,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;QAE/D,IAAI,CAAC,UAAU,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,sBAAsB,IAAI,CAAC,UAAU,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAA;QAC1F,CAAC;QAED,IAAI,UAAU,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,2BAA2B,IAAI,CAAC,UAAU,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAA;QAC/F,CAAC;QAED,MAAM,kBAAkB,GAAG,eAAe,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAA;QAEhE,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACzB,qBAAqB,CAAC,kBAAkB,EAAE,SAAS,CAAC,CAAA;QACrD,CAAC;aAAM,CAAC;YACP,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE,KAAK,CAAC,CAAA;YAC1D,MAAM,OAAO,GAAG,mBAAmB,CAAC,kBAAkB,EAAE,SAAS,CAAC,CAAA;YAElE,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,UAAmC,CAAC,CAAA;gBAClE,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;oBAClB,qBAAqB,CAAC,kBAAkB,EAAE,GAAG,SAAS,IAAI,KAAK,EAAE,CAAC,CAAA;gBACnE,CAAC;qBAAM,CAAC;oBACP,OAAO,QAAQ,CAAA;gBAChB,CAAC;YACF,CAAC;iBAAM,IAAI,OAAO,KAAK,UAAU,EAAE,CAAC;gBACnC,qBAAqB,CAAC,kBAAkB,EAAE,SAAS,CAAC,CAAA;YACrD,CAAC;QACF,CAAC;QAED,OAAO,iBAAiB,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAA;IAC/F,CAAC;CACD;AAED,eAAe,KAAK,CAAA","sourcesContent":["import { updateCache, addCache, getCache } from './cache.js'\nimport { cpus } from 'os'\nimport path from 'path'\nimport slugify from '@sindresorhus/slugify'\nimport { Readable } from 'stream'\nimport {\n\tdatabaseValueToPieceFrontmatterValue,\n\tgetPieceFrontmatterSchemaFields,\n\tinitializePieceFrontMatter,\n\tPieceFrontmatter,\n\tPieceFrontmatterSchema,\n\tPieceFrontmatterSchemaField,\n\tPieceFrontMatterValue,\n} from './utils/frontmatter.js'\nimport {\n\tfindFrontmatterField,\n\tsetFrontmatterValue,\n\tunsetFrontmatterValue,\n\tgetFrontmatterValue,\n} from './utils/frontmatter.path.js'\nimport LuzzleStorage from '../storage/abstract.js'\nimport { makePieceMarkdown, makePieceMarkdownString, PieceMarkdown } from './utils/markdown.js'\nimport {\n\tisAttachableStream,\n\tcalculateHashFromFile,\n\tmakePieceAttachment,\n\tmakePieceValue,\n} from './utils/piece.js'\nimport { LuzzleDatabase, LuzzleSelectable } from '../database/tables/index.js'\nimport compile from '../lib/ajv.js'\nimport {\n\tgetValidatePieceItemErrors,\n\tmakePieceItemInsertable,\n\tmakePieceItemUpdatable,\n\tvalidatePieceItem,\n} from './item.js'\nimport { extractFullMarkdown } from '../lib/markdown.js'\nimport { deleteItem, insertItem, selectItem, selectItems, updateItem } from './items.js'\nimport { LUZZLE_PIECE_FILE_EXTENSION } from './assets.js'\n\nexport interface InterfacePiece<F extends PieceFrontmatter> {\n\tnew(directory: string, pieceName: string, schemaOverride?: PieceFrontmatterSchema<F>): Piece<F>\n}\n\nexport type PiecePruneResult =\n\t| {\n\t\taction: 'pruned'\n\t\tfile: string\n\t\terror?: false\n\t}\n\t| {\n\t\tfile: string\n\t\terror: true\n\t\tmessage: string\n\t}\n\nexport type PieceSyncResult =\n\t| {\n\t\taction: 'added' | 'updated' | 'skipped'\n\t\tfile: string\n\t\terror?: false\n\t}\n\t| {\n\t\tfile: string\n\t\terror: true\n\t\tmessage: string\n\t}\n\nclass Piece<F extends PieceFrontmatter> {\n\tprivate _validator?: ReturnType<typeof compile<F>>\n\tprivate _schema: PieceFrontmatterSchema<F>\n\tprivate _storage: LuzzleStorage\n\tprivate _pieceName: string\n\tprivate _fields?: Array<PieceFrontmatterSchemaField>\n\n\tconstructor(pieceName: string, storage: LuzzleStorage, schema: PieceFrontmatterSchema<F>) {\n\t\tthis._storage = storage\n\t\tthis._schema = schema\n\t\tthis._pieceName = pieceName\n\n\t\tif (this._pieceName !== this._schema.title) {\n\t\t\tthrow new Error(`${pieceName} does not match the schema title: ${this._schema.title}`)\n\t\t}\n\t}\n\n\tasync create(directory: string, name: string): Promise<PieceMarkdown<F>> {\n\t\tconst slug = slugify(name)\n\t\tconst filename = `${slug}.${this.type}.${LUZZLE_PIECE_FILE_EXTENSION}`\n\t\tconst file = path.join(directory, filename)\n\t\tconst exists = await this._storage.exists(file)\n\n\t\tif (exists) {\n\t\t\tthrow new Error(`file already exists: ${file}`)\n\t\t}\n\n\t\tconst frontmatter = initializePieceFrontMatter(this._schema, true) as F\n\t\treturn makePieceMarkdown(file, this._pieceName, undefined, frontmatter)\n\t}\n\n\tasync delete(file: string) {\n\t\tconst exists = await this._storage.exists(file)\n\n\t\tif (exists) {\n\t\t\tawait this._storage.delete(file)\n\t\t\treturn\n\t\t}\n\n\t\tthrow new Error(`${file} does not exist`)\n\t}\n\n\tget type() {\n\t\treturn this._pieceName\n\t}\n\n\tget schema() {\n\t\treturn this._schema\n\t}\n\n\tprotected get validator(): ReturnType<typeof compile<F>> {\n\t\tthis._validator = this._validator || compile<F>(this._schema)\n\t\treturn this._validator\n\t}\n\n\tget fields() {\n\t\tthis._fields = this._fields || getPieceFrontmatterSchemaFields(this._schema)\n\t\treturn this._fields\n\t}\n\n\tasync isOutdated(file: string, db: LuzzleDatabase): Promise<boolean> {\n\t\tconst fileStat = await this._storage.stat(file).catch(() => null)\n\n\t\tif (fileStat) {\n\t\t\tconst cache = await getCache(db, file)\n\t\t\tconst cachedDate = cache?.date_updated || cache?.date_added\n\n\t\t\treturn !cachedDate || fileStat.last_modified.getTime() > cachedDate\n\t\t}\n\n\t\tthrow new Error(`${file} does not exist`)\n\t}\n\n\tvalidate(markdown: PieceMarkdown<F>): { isValid: true } | { isValid: false; errors: string[] } {\n\t\tconst valid = validatePieceItem(markdown, this.validator)\n\n\t\tif (valid) {\n\t\t\treturn { isValid: true }\n\t\t} else {\n\t\t\tconst errors = getValidatePieceItemErrors(this.validator)\n\t\t\treturn { isValid: false, errors }\n\t\t}\n\t}\n\n\tasync get(file: string): Promise<PieceMarkdown<F>> {\n\t\tconst exists = await this._storage.exists(file)\n\n\t\tif (exists) {\n\t\t\tconst contents = await this._storage.readFile(file, 'text')\n\t\t\tconst data = await extractFullMarkdown(contents)\n\n\t\t\tif (!/^\\//.test(file)) {\n\t\t\t\treturn makePieceMarkdown(file, this._pieceName, data.markdown, data.frontmatter as F)\n\t\t\t}\n\t\t}\n\n\t\tthrow new Error(`${file} does not exist`)\n\t}\n\n\tasync write(markdown: PieceMarkdown<F>): Promise<void> {\n\t\tconst validated = this.validate(markdown)\n\n\t\tif (validated.isValid) {\n\t\t\tconst markdownString = makePieceMarkdownString(markdown)\n\t\t\tawait this._storage.writeFile(markdown.filePath, markdownString)\n\t\t} else {\n\t\t\tthrow new Error(\n\t\t\t\t`Could not write ${markdown.filePath} due to\\n\\n: ${validated.errors.join('\\n')}`\n\t\t\t)\n\t\t}\n\t}\n\n\tasync prune(db: LuzzleDatabase, files: string[], options?: { dryRun: boolean }) {\n\t\tconst dbPieces = await selectItems(db, { type: this._pieceName })\n\t\tconst diskPiecesSet = new Set<string>(files)\n\t\tconst missingPieces = dbPieces.filter((piece) => !diskPiecesSet.has(piece.file_path))\n\t\tconst missingFiles = missingPieces.map((piece) => piece.file_path)\n\t\tconst stream = Readable.from(missingFiles)\n\n\t\treturn stream.map(\n\t\t\tasync (file: string): Promise<PiecePruneResult> => {\n\t\t\t\ttry {\n\t\t\t\t\tif (!options?.dryRun) {\n\t\t\t\t\t\tawait deleteItem(db, file)\n\t\t\t\t\t}\n\t\t\t\t\treturn { action: 'pruned', file }\n\t\t\t\t} catch (error) {\n\t\t\t\t\treturn { file, error: true, message: `error pruning piece: ${error}` }\n\t\t\t\t}\n\t\t\t},\n\t\t\t{ concurrency: cpus().length }\n\t\t) as Readable & AsyncIterable<PiecePruneResult>\n\t}\n\n\tasync sync(db: LuzzleDatabase, files: string[], options?: { dryRun?: boolean; force?: boolean }) {\n\t\tconst stream = Readable.from(files)\n\n\t\treturn stream.map(\n\t\t\tasync (file: string): Promise<PieceSyncResult> => {\n\t\t\t\ttry {\n\t\t\t\t\tconst markdown = await this.get(file)\n\t\t\t\t\tconst dbPiece = await selectItem(db, markdown.filePath)\n\n\t\t\t\t\tif (dbPiece) {\n\t\t\t\t\t\tconst readStream = this._storage.createReadStream(markdown.filePath)\n\t\t\t\t\t\tconst newHash = await calculateHashFromFile(readStream)\n\t\t\t\t\t\tconst cache = await getCache(db, markdown.filePath)\n\t\t\t\t\t\tif (options?.force || cache?.content_hash !== newHash) {\n\t\t\t\t\t\t\tif (!options?.dryRun) {\n\t\t\t\t\t\t\t\tawait this.syncMarkdownUpdate(db, markdown, dbPiece)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\treturn { action: 'updated', file }\n\t\t\t\t\t\t}\n\t\t\t\t\t\t\n\t\t\t\t\t\tif (!options?.dryRun) {\n\t\t\t\t\t\t\tawait updateCache(db, markdown.filePath, newHash)\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\treturn { action: 'skipped', file }\n\t\t\t\t\t} else {\n\t\t\t\t\t\tif (!options?.dryRun) {\n\t\t\t\t\t\t\tawait this.syncMarkdownAdd(db, markdown)\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn { action: 'added', file }\n\t\t\t\t\t}\n\t\t\t\t} catch (error) {\n\t\t\t\t\treturn { file, error: true, message: `error syncing piece: ${error}` }\n\t\t\t\t}\n\t\t\t},\n\t\t\t{ concurrency: cpus().length }\n\t\t) as Readable & AsyncIterable<PieceSyncResult>\n\t}\n\n\tasync syncMarkdownAdd(db: LuzzleDatabase, markdown: PieceMarkdown<F>): Promise<void> {\n\t\tconst createInput = makePieceItemInsertable(this._pieceName, markdown, this._schema)\n\t\tconst readStream = this._storage.createReadStream(markdown.filePath)\n\t\tconst hash = await calculateHashFromFile(readStream)\n\n\t\tawait insertItem(db, createInput)\n\t\tawait addCache(db, markdown.filePath, hash)\n\t}\n\n\tasync syncMarkdown(db: LuzzleDatabase, markdown: PieceMarkdown<F>): Promise<void> {\n\t\tconst dbPiece = await selectItem(db, markdown.filePath)\n\n\t\tif (dbPiece) {\n\t\t\tawait this.syncMarkdownUpdate(db, markdown, dbPiece)\n\t\t} else {\n\t\t\tawait this.syncMarkdownAdd(db, markdown)\n\t\t}\n\t}\n\n\tasync syncMarkdownUpdate(\n\t\tdb: LuzzleDatabase,\n\t\tmarkdown: PieceMarkdown<F>,\n\t\tdata: LuzzleSelectable<'pieces_items'>\n\t): Promise<void> {\n\t\tconst updateInput = makePieceItemUpdatable(markdown, this._schema, data, false)\n\t\tawait updateItem(db, markdown.filePath, updateInput)\n\t\tconst readStream = this._storage.createReadStream(markdown.filePath)\n\t\tconst hash = await calculateHashFromFile(readStream)\n\n\t\tawait updateCache(db, data.file_path, hash)\n\t}\n\n\ttoMarkdown(data: LuzzleSelectable<'pieces_items'>): PieceMarkdown<F> {\n\t\tconst frontmatter = JSON.parse(data.frontmatter_json)\n\t\tconst frontmatterJson: Record<string, unknown> = {}\n\t\tconst dataKeys = Object.keys(frontmatter)\n\t\tconst fields = getPieceFrontmatterSchemaFields(this._schema).filter(\n\t\t\t(f): f is PieceFrontmatterSchemaField & { name: string } =>\n\t\t\t\t!!f.name && dataKeys.includes(f.name)\n\t\t)\n\n\t\tfields.forEach((field) => {\n\t\t\tconst name = field.name\n\t\t\tconst value = frontmatter[name]\n\n\t\t\tfrontmatterJson[name] = databaseValueToPieceFrontmatterValue(value, field)\n\t\t})\n\n\t\treturn makePieceMarkdown(data.file_path, data.type, data.note_markdown, frontmatterJson as F)\n\t}\n\n\tgetField(markdown: PieceMarkdown<F>, fieldPath: string): PieceFrontMatterValue | undefined {\n\t\treturn getFrontmatterValue(markdown.frontmatter, fieldPath)\n\t}\n\n\tasync setFields(\n\t\tmarkdown: PieceMarkdown<F>,\n\t\tfields: Record<string, unknown>\n\t): Promise<PieceMarkdown<F>> {\n\t\tlet updatedMarkdown = markdown\n\n\t\tfor (const fieldname in fields) {\n\t\t\tupdatedMarkdown = await this.setField(updatedMarkdown, fieldname, fields[fieldname])\n\t\t}\n\n\t\treturn updatedMarkdown\n\t}\n\n\tasync removeFields(\n\t\tmarkdown: PieceMarkdown<F>,\n\t\tfields: string[]\n\t): Promise<PieceMarkdown<Omit<F, keyof F>>> {\n\t\tlet updatedMarkdown = markdown as unknown as PieceMarkdown<Omit<F, keyof F>>\n\n\t\tfor (const field of fields) {\n\t\t\tupdatedMarkdown = (await this.removeField(\n\t\t\t\tupdatedMarkdown as unknown as PieceMarkdown<F>,\n\t\t\t\tfield\n\t\t\t)) as unknown as PieceMarkdown<Omit<F, keyof F>>\n\t\t}\n\n\t\treturn updatedMarkdown\n\t}\n\n\tasync setField(\n\t\tmarkdown: PieceMarkdown<F>,\n\t\tfieldPath: string,\n\t\tvalue: unknown\n\t): Promise<PieceMarkdown<F>> {\n\t\tconst pieceField = findFrontmatterField(this.fields, fieldPath)\n\n\t\tif (!pieceField) {\n\t\t\tthrow new Error(`${fieldPath} is not a field in ${this._pieceName} ${markdown.filePath}`)\n\t\t}\n\n\t\tconst isArray = pieceField.type === 'array'\n\t\tconst itemField = isArray ? { ...pieceField.items, name: pieceField.name } : pieceField\n\t\tconst values = Array.isArray(value) ? value : [value]\n\n\t\tconst updatedFrontmatter = structuredClone(markdown.frontmatter)\n\n\t\ttry {\n\t\t\tif (isArray) {\n\t\t\t\tconst current = getFrontmatterValue(updatedFrontmatter, fieldPath)\n\t\t\t\tif (!Array.isArray(current)) {\n\t\t\t\t\tsetFrontmatterValue(updatedFrontmatter, fieldPath, [] as unknown as PieceFrontMatterValue)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tfor (const one of values) {\n\t\t\t\tconst pieceValue = await makePieceValue(itemField, one)\n\n\t\t\t\tconst val = isAttachableStream(pieceValue)\n\t\t\t\t\t? await makePieceAttachment(\n\t\t\t\t\t\tmarkdown.filePath,\n\t\t\t\t\t\titemField as PieceFrontmatterSchemaField,\n\t\t\t\t\t\tpieceValue,\n\t\t\t\t\t\tthis._storage\n\t\t\t\t\t)\n\t\t\t\t\t: (pieceValue as PieceFrontMatterValue)\n\n\t\t\t\tsetFrontmatterValue(updatedFrontmatter, fieldPath, val)\n\t\t\t}\n\t\t} catch (e) {\n\t\t\tconst error = e as Error\n\t\t\tconsole.error(`could not set field ${fieldPath}: ${error.message}`)\n\t\t\treturn markdown\n\t\t}\n\n\t\treturn makePieceMarkdown(markdown.filePath, markdown.piece, markdown.note, updatedFrontmatter)\n\t}\n\n\tasync removeField(\n\t\tmarkdown: PieceMarkdown<F>,\n\t\tfieldPath: string,\n\t\tvalue?: number | string | boolean\n\t): Promise<PieceMarkdown<F>> {\n\t\tconst pieceField = findFrontmatterField(this.fields, fieldPath)\n\n\t\tif (!pieceField) {\n\t\t\tthrow new Error(`${fieldPath} is not a field in ${this._pieceName} ${markdown.filePath}`)\n\t\t}\n\n\t\tif (pieceField.nullable !== true) {\n\t\t\tthrow new Error(`${fieldPath} is a required field in ${this._pieceName} ${markdown.filePath}`)\n\t\t}\n\n\t\tconst updatedFrontmatter = structuredClone(markdown.frontmatter)\n\n\t\tif (value === undefined) {\n\t\t\tunsetFrontmatterValue(updatedFrontmatter, fieldPath)\n\t\t} else {\n\t\t\tconst pieceValue = await makePieceValue(pieceField, value)\n\t\t\tconst current = getFrontmatterValue(updatedFrontmatter, fieldPath)\n\n\t\t\tif (Array.isArray(current) && !isAttachableStream(pieceValue)) {\n\t\t\t\tconst index = current.indexOf(pieceValue as PieceFrontMatterValue)\n\t\t\t\tif (index !== -1) {\n\t\t\t\t\tunsetFrontmatterValue(updatedFrontmatter, `${fieldPath}.${index}`)\n\t\t\t\t} else {\n\t\t\t\t\treturn markdown\n\t\t\t\t}\n\t\t\t} else if (current === pieceValue) {\n\t\t\t\tunsetFrontmatterValue(updatedFrontmatter, fieldPath)\n\t\t\t}\n\t\t}\n\n\t\treturn makePieceMarkdown(markdown.filePath, markdown.piece, markdown.note, updatedFrontmatter)\n\t}\n}\n\nexport default Piece\n"]}
+{"version":3,"file":"Piece.js","sourceRoot":"","sources":["../../../src/pieces/Piece.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AAC5D,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAA;AACzB,OAAO,IAAI,MAAM,MAAM,CAAA;AACvB,OAAO,OAAO,MAAM,uBAAuB,CAAA;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAA;AACjC,OAAO,EACN,oCAAoC,EACpC,+BAA+B,EAC/B,0BAA0B,GAK1B,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EACN,oBAAoB,EACpB,mBAAmB,EACnB,qBAAqB,EACrB,mBAAmB,GACnB,MAAM,6BAA6B,CAAA;AAEpC,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAiB,MAAM,qBAAqB,CAAA;AAC/F,OAAO,EACN,kBAAkB,EAClB,qBAAqB,EACrB,mBAAmB,EACnB,cAAc,GACd,MAAM,kBAAkB,CAAA;AAEzB,OAAO,OAAO,MAAM,eAAe,CAAA;AACnC,OAAO,EACN,0BAA0B,EAC1B,uBAAuB,EACvB,sBAAsB,EACtB,iBAAiB,GACjB,MAAM,WAAW,CAAA;AAClB,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AACxF,OAAO,EAAE,2BAA2B,EAAE,MAAM,aAAa,CAAA;AA8BzD,MAAM,KAAK;IACF,UAAU,CAAgC;IAC1C,OAAO,CAA2B;IAClC,QAAQ,CAAe;IACvB,UAAU,CAAQ;IAClB,OAAO,CAAqC;IAEpD,YAAY,SAAiB,EAAE,OAAsB,EAAE,MAAiC;QACvF,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAA;QACvB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAA;QAE3B,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,qCAAqC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;QACvF,CAAC;IACF,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,SAAiB,EAAE,IAAY;QAC3C,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;QAC1B,MAAM,QAAQ,GAAG,GAAG,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,2BAA2B,EAAE,CAAA;QACtE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;QAC3C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAE/C,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,EAAE,CAAC,CAAA;QAChD,CAAC;QAED,MAAM,WAAW,GAAG,0BAA0B,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAM,CAAA;QACvE,OAAO,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,SAAS,EAAE,WAAW,CAAC,CAAA;IACxE,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAY;QACxB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAE/C,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAChC,OAAM;QACP,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,iBAAiB,CAAC,CAAA;IAC1C,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,UAAU,CAAA;IACvB,CAAC;IAED,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,OAAO,CAAA;IACpB,CAAC;IAED,IAAc,SAAS;QACtB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,OAAO,CAAI,IAAI,CAAC,OAAO,CAAC,CAAA;QAC7D,OAAO,IAAI,CAAC,UAAU,CAAA;IACvB,CAAC;IAED,IAAI,MAAM;QACT,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,+BAA+B,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC5E,OAAO,IAAI,CAAC,OAAO,CAAA;IACpB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAY,EAAE,EAAkB;QAChD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAA;QAEjE,IAAI,QAAQ,EAAE,CAAC;YACd,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;YACtC,MAAM,UAAU,GAAG,KAAK,EAAE,YAAY,IAAI,KAAK,EAAE,UAAU,CAAA;YAE3D,OAAO,CAAC,UAAU,IAAI,QAAQ,CAAC,aAAa,CAAC,OAAO,EAAE,GAAG,UAAU,CAAA;QACpE,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,iBAAiB,CAAC,CAAA;IAC1C,CAAC;IAED,QAAQ,CAAC,QAA0B;QAClC,MAAM,KAAK,GAAG,iBAAiB,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAA;QAEzD,IAAI,KAAK,EAAE,CAAC;YACX,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;QACzB,CAAC;aAAM,CAAC;YACP,MAAM,MAAM,GAAG,0BAA0B,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACzD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAA;QAClC,CAAC;IACF,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAY;QACrB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAE/C,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;YAC3D,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAAC,QAAQ,CAAC,CAAA;YAEhD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvB,OAAO,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAgB,CAAC,CAAA;YACtF,CAAC;QACF,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,iBAAiB,CAAC,CAAA;IAC1C,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAA0B;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAEzC,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;YACvB,MAAM,cAAc,GAAG,uBAAuB,CAAC,QAAQ,CAAC,CAAA;YACxD,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAA;QACjE,CAAC;aAAM,CAAC;YACP,MAAM,IAAI,KAAK,CACd,mBAAmB,QAAQ,CAAC,QAAQ,gBAAgB,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACjF,CAAA;QACF,CAAC;IACF,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAkB,EAAE,KAAe,EAAE,OAA6B;QAC7E,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAA;QACjE,MAAM,aAAa,GAAG,IAAI,GAAG,CAAS,KAAK,CAAC,CAAA;QAC5C,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAA;QACrF,MAAM,YAAY,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;QAClE,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;QAE1C,OAAO,MAAM,CAAC,GAAG,CAChB,KAAK,EAAE,IAAY,EAA6B,EAAE;YACjD,IAAI,CAAC;gBACJ,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;oBACtB,MAAM,UAAU,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;gBAC3B,CAAC;gBACD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;YAClC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,wBAAwB,KAAK,EAAE,EAAE,CAAA;YACvE,CAAC;QACF,CAAC,EACD,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,MAAM,EAAE,CACgB,CAAA;IAChD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAkB,EAAE,KAAe,EAAE,OAA+C;QAC9F,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAEnC,OAAO,MAAM,CAAC,GAAG,CAChB,KAAK,EAAE,IAAY,EAA4B,EAAE;YAChD,IAAI,CAAC;gBACJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;gBACrC,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAA;gBAEvD,IAAI,OAAO,EAAE,CAAC;oBACb,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;oBACpE,MAAM,OAAO,GAAG,MAAM,qBAAqB,CAAC,UAAU,CAAC,CAAA;oBACvD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAA;oBACnD,IAAI,OAAO,EAAE,KAAK,IAAI,KAAK,EAAE,YAAY,KAAK,OAAO,EAAE,CAAC;wBACvD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;4BACtB,MAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAA;wBACrD,CAAC;wBACD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAA;oBACnC,CAAC;oBAED,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;wBACtB,MAAM,WAAW,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;oBAClD,CAAC;oBAED,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAA;gBACnC,CAAC;qBAAM,CAAC;oBACP,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;wBACtB,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAA;oBACzC,CAAC;oBACD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;gBACjC,CAAC;YACF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,wBAAwB,KAAK,EAAE,EAAE,CAAA;YACvE,CAAC;QACF,CAAC,EACD,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,MAAM,EAAE,CACe,CAAA;IAC/C,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,EAAkB,EAAE,QAA0B;QACnE,MAAM,WAAW,GAAG,uBAAuB,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;QACpF,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;QACpE,MAAM,IAAI,GAAG,MAAM,qBAAqB,CAAC,UAAU,CAAC,CAAA;QAEpD,MAAM,UAAU,CAAC,EAAE,EAAE,WAAW,CAAC,CAAA;QACjC,MAAM,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;IAC5C,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,EAAkB,EAAE,QAA0B;QAChE,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAEvD,IAAI,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAA;QACrD,CAAC;aAAM,CAAC;YACP,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAA;QACzC,CAAC;IACF,CAAC;IAED,KAAK,CAAC,kBAAkB,CACvB,EAAkB,EAClB,QAA0B,EAC1B,IAAsC;QAEtC,MAAM,WAAW,GAAG,sBAAsB,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAA;QAC/E,MAAM,UAAU,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;QACpD,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;QACpE,MAAM,IAAI,GAAG,MAAM,qBAAqB,CAAC,UAAU,CAAC,CAAA;QAEpD,MAAM,WAAW,CAAC,EAAE,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IAC5C,CAAC;IAED,UAAU,CAAC,IAAsC;QAChD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QACrD,MAAM,eAAe,GAA4B,EAAE,CAAA;QACnD,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;QACzC,MAAM,MAAM,GAAG,+BAA+B,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,CAClE,CAAC,CAAC,EAAuD,EAAE,CAC1D,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CACtC,CAAA;QAED,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;YACxB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAA;YACvB,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAA;YAE/B,eAAe,CAAC,IAAI,CAAC,GAAG,oCAAoC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;QAC3E,CAAC,CAAC,CAAA;QAEF,OAAO,iBAAiB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,aAAa,EAAE,eAAoB,CAAC,CAAA;IAC9F,CAAC;IAED,QAAQ,CAAC,QAA0B,EAAE,SAAiB;QACrD,OAAO,mBAAmB,CAAC,QAAQ,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;IAC5D,CAAC;IAED,KAAK,CAAC,SAAS,CACd,QAA0B,EAC1B,MAA+B;QAE/B,IAAI,eAAe,GAAG,QAAQ,CAAA;QAE9B,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;YAChC,eAAe,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,eAAe,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAA;QACrF,CAAC;QAED,OAAO,eAAe,CAAA;IACvB,CAAC;IAED,KAAK,CAAC,YAAY,CACjB,QAA0B,EAC1B,MAAgB;QAEhB,IAAI,eAAe,GAAG,QAAsD,CAAA;QAE5E,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC5B,eAAe,GAAG,CAAC,MAAM,IAAI,CAAC,WAAW,CACxC,eAA8C,EAC9C,KAAK,CACL,CAA+C,CAAA;QACjD,CAAC;QAED,OAAO,eAAe,CAAA;IACvB,CAAC;IAED,KAAK,CAAC,QAAQ,CACb,QAA0B,EAC1B,SAAiB,EACjB,KAAc;QAEd,MAAM,UAAU,GAAG,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;QAE/D,IAAI,CAAC,UAAU,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,sBAAsB,IAAI,CAAC,UAAU,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAA;QAC1F,CAAC;QAED,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,KAAK,OAAO,CAAA;QAC3C,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,UAAU,CAAA;QACvF,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;QAErD,MAAM,kBAAkB,GAAG,eAAe,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAA;QAEhE,IAAI,CAAC;YACJ,IAAI,OAAO,EAAE,CAAC;gBACb,MAAM,OAAO,GAAG,mBAAmB,CAAC,kBAAkB,EAAE,SAAS,CAAC,CAAA;gBAClE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC7B,mBAAmB,CAAC,kBAAkB,EAAE,SAAS,EAAE,EAAsC,CAAC,CAAA;gBAC3F,CAAC;YACF,CAAC;YAED,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;gBAC1B,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;gBAEvD,MAAM,GAAG,GAAG,kBAAkB,CAAC,UAAU,CAAC;oBACzC,CAAC,CAAC,MAAM,mBAAmB,CAC1B,QAAQ,CAAC,QAAQ,EACjB,SAAwC,EACxC,UAAU,EACV,IAAI,CAAC,QAAQ,CACb;oBACD,CAAC,CAAE,UAAoC,CAAA;gBAExC,mBAAmB,CAAC,kBAAkB,EAAE,SAAS,EAAE,GAAG,CAAC,CAAA;YACxD,CAAC;QACF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACZ,MAAM,KAAK,GAAG,CAAU,CAAA;YACxB,OAAO,CAAC,KAAK,CAAC,uBAAuB,SAAS,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAA;YACnE,OAAO,QAAQ,CAAA;QAChB,CAAC;QAED,OAAO,iBAAiB,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAA;IAC/F,CAAC;IAED,KAAK,CAAC,WAAW,CAChB,QAA0B,EAC1B,SAAiB,EACjB,KAAiC;QAEjC,MAAM,UAAU,GAAG,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;QAE/D,IAAI,CAAC,UAAU,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,sBAAsB,IAAI,CAAC,UAAU,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAA;QAC1F,CAAC;QAED,IAAI,UAAU,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,2BAA2B,IAAI,CAAC,UAAU,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAA;QAC/F,CAAC;QAED,MAAM,kBAAkB,GAAG,eAAe,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAA;QAEhE,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACzB,qBAAqB,CAAC,kBAAkB,EAAE,SAAS,CAAC,CAAA;QACrD,CAAC;aAAM,CAAC;YACP,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE,KAAK,CAAC,CAAA;YAC1D,MAAM,OAAO,GAAG,mBAAmB,CAAC,kBAAkB,EAAE,SAAS,CAAC,CAAA;YAElE,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,UAAmC,CAAC,CAAA;gBAClE,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;oBAClB,qBAAqB,CAAC,kBAAkB,EAAE,GAAG,SAAS,IAAI,KAAK,EAAE,CAAC,CAAA;gBACnE,CAAC;qBAAM,CAAC;oBACP,OAAO,QAAQ,CAAA;gBAChB,CAAC;YACF,CAAC;iBAAM,IAAI,OAAO,KAAK,UAAU,EAAE,CAAC;gBACnC,qBAAqB,CAAC,kBAAkB,EAAE,SAAS,CAAC,CAAA;YACrD,CAAC;QACF,CAAC;QAED,OAAO,iBAAiB,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAA;IAC/F,CAAC;CACD;AAED,eAAe,KAAK,CAAA","sourcesContent":["import { updateCache, addCache, getCache } from './cache.js'\nimport { cpus } from 'os'\nimport path from 'path'\nimport slugify from '@sindresorhus/slugify'\nimport { Readable } from 'stream'\nimport {\n\tdatabaseValueToPieceFrontmatterValue,\n\tgetPieceFrontmatterSchemaFields,\n\tinitializePieceFrontMatter,\n\tPieceFrontmatter,\n\tPieceFrontmatterSchema,\n\tPieceFrontmatterSchemaField,\n\tPieceFrontMatterValue,\n} from './utils/frontmatter.js'\nimport {\n\tfindFrontmatterField,\n\tsetFrontmatterValue,\n\tunsetFrontmatterValue,\n\tgetFrontmatterValue,\n} from './utils/frontmatter.path.js'\nimport LuzzleStorage from '../storage/abstract.js'\nimport { makePieceMarkdown, makePieceMarkdownString, PieceMarkdown } from './utils/markdown.js'\nimport {\n\tisAttachableStream,\n\tcalculateHashFromFile,\n\tsavePieceFieldAsset,\n\tmakePieceValue,\n} from './utils/piece.js'\nimport { LuzzleDatabase, LuzzleSelectable } from '../database/tables/index.js'\nimport compile from '../lib/ajv.js'\nimport {\n\tgetValidatePieceItemErrors,\n\tmakePieceItemInsertable,\n\tmakePieceItemUpdatable,\n\tvalidatePieceItem,\n} from './item.js'\nimport { extractFullMarkdown } from '../lib/markdown.js'\nimport { deleteItem, insertItem, selectItem, selectItems, updateItem } from './items.js'\nimport { LUZZLE_PIECE_FILE_EXTENSION } from './assets.js'\n\nexport interface InterfacePiece<F extends PieceFrontmatter> {\n\tnew(directory: string, pieceName: string, schemaOverride?: PieceFrontmatterSchema<F>): Piece<F>\n}\n\nexport type PiecePruneResult =\n\t| {\n\t\taction: 'pruned'\n\t\tfile: string\n\t\terror?: false\n\t}\n\t| {\n\t\tfile: string\n\t\terror: true\n\t\tmessage: string\n\t}\n\nexport type PieceSyncResult =\n\t| {\n\t\taction: 'added' | 'updated' | 'skipped'\n\t\tfile: string\n\t\terror?: false\n\t}\n\t| {\n\t\tfile: string\n\t\terror: true\n\t\tmessage: string\n\t}\n\nclass Piece<F extends PieceFrontmatter> {\n\tprivate _validator?: ReturnType<typeof compile<F>>\n\tprivate _schema: PieceFrontmatterSchema<F>\n\tprivate _storage: LuzzleStorage\n\tprivate _pieceName: string\n\tprivate _fields?: Array<PieceFrontmatterSchemaField>\n\n\tconstructor(pieceName: string, storage: LuzzleStorage, schema: PieceFrontmatterSchema<F>) {\n\t\tthis._storage = storage\n\t\tthis._schema = schema\n\t\tthis._pieceName = pieceName\n\n\t\tif (this._pieceName !== this._schema.title) {\n\t\t\tthrow new Error(`${pieceName} does not match the schema title: ${this._schema.title}`)\n\t\t}\n\t}\n\n\tasync create(directory: string, name: string): Promise<PieceMarkdown<F>> {\n\t\tconst slug = slugify(name)\n\t\tconst filename = `${slug}.${this.type}.${LUZZLE_PIECE_FILE_EXTENSION}`\n\t\tconst file = path.join(directory, filename)\n\t\tconst exists = await this._storage.exists(file)\n\n\t\tif (exists) {\n\t\t\tthrow new Error(`file already exists: ${file}`)\n\t\t}\n\n\t\tconst frontmatter = initializePieceFrontMatter(this._schema, true) as F\n\t\treturn makePieceMarkdown(file, this._pieceName, undefined, frontmatter)\n\t}\n\n\tasync delete(file: string) {\n\t\tconst exists = await this._storage.exists(file)\n\n\t\tif (exists) {\n\t\t\tawait this._storage.delete(file)\n\t\t\treturn\n\t\t}\n\n\t\tthrow new Error(`${file} does not exist`)\n\t}\n\n\tget type() {\n\t\treturn this._pieceName\n\t}\n\n\tget schema() {\n\t\treturn this._schema\n\t}\n\n\tprotected get validator(): ReturnType<typeof compile<F>> {\n\t\tthis._validator = this._validator || compile<F>(this._schema)\n\t\treturn this._validator\n\t}\n\n\tget fields() {\n\t\tthis._fields = this._fields || getPieceFrontmatterSchemaFields(this._schema)\n\t\treturn this._fields\n\t}\n\n\tasync isOutdated(file: string, db: LuzzleDatabase): Promise<boolean> {\n\t\tconst fileStat = await this._storage.stat(file).catch(() => null)\n\n\t\tif (fileStat) {\n\t\t\tconst cache = await getCache(db, file)\n\t\t\tconst cachedDate = cache?.date_updated || cache?.date_added\n\n\t\t\treturn !cachedDate || fileStat.last_modified.getTime() > cachedDate\n\t\t}\n\n\t\tthrow new Error(`${file} does not exist`)\n\t}\n\n\tvalidate(markdown: PieceMarkdown<F>): { isValid: true } | { isValid: false; errors: string[] } {\n\t\tconst valid = validatePieceItem(markdown, this.validator)\n\n\t\tif (valid) {\n\t\t\treturn { isValid: true }\n\t\t} else {\n\t\t\tconst errors = getValidatePieceItemErrors(this.validator)\n\t\t\treturn { isValid: false, errors }\n\t\t}\n\t}\n\n\tasync get(file: string): Promise<PieceMarkdown<F>> {\n\t\tconst exists = await this._storage.exists(file)\n\n\t\tif (exists) {\n\t\t\tconst contents = await this._storage.readFile(file, 'text')\n\t\t\tconst data = await extractFullMarkdown(contents)\n\n\t\t\tif (!/^\\//.test(file)) {\n\t\t\t\treturn makePieceMarkdown(file, this._pieceName, data.markdown, data.frontmatter as F)\n\t\t\t}\n\t\t}\n\n\t\tthrow new Error(`${file} does not exist`)\n\t}\n\n\tasync write(markdown: PieceMarkdown<F>): Promise<void> {\n\t\tconst validated = this.validate(markdown)\n\n\t\tif (validated.isValid) {\n\t\t\tconst markdownString = makePieceMarkdownString(markdown)\n\t\t\tawait this._storage.writeFile(markdown.filePath, markdownString)\n\t\t} else {\n\t\t\tthrow new Error(\n\t\t\t\t`Could not write ${markdown.filePath} due to\\n\\n: ${validated.errors.join('\\n')}`\n\t\t\t)\n\t\t}\n\t}\n\n\tasync prune(db: LuzzleDatabase, files: string[], options?: { dryRun: boolean }) {\n\t\tconst dbPieces = await selectItems(db, { type: this._pieceName })\n\t\tconst diskPiecesSet = new Set<string>(files)\n\t\tconst missingPieces = dbPieces.filter((piece) => !diskPiecesSet.has(piece.file_path))\n\t\tconst missingFiles = missingPieces.map((piece) => piece.file_path)\n\t\tconst stream = Readable.from(missingFiles)\n\n\t\treturn stream.map(\n\t\t\tasync (file: string): Promise<PiecePruneResult> => {\n\t\t\t\ttry {\n\t\t\t\t\tif (!options?.dryRun) {\n\t\t\t\t\t\tawait deleteItem(db, file)\n\t\t\t\t\t}\n\t\t\t\t\treturn { action: 'pruned', file }\n\t\t\t\t} catch (error) {\n\t\t\t\t\treturn { file, error: true, message: `error pruning piece: ${error}` }\n\t\t\t\t}\n\t\t\t},\n\t\t\t{ concurrency: cpus().length }\n\t\t) as Readable & AsyncIterable<PiecePruneResult>\n\t}\n\n\tasync sync(db: LuzzleDatabase, files: string[], options?: { dryRun?: boolean; force?: boolean }) {\n\t\tconst stream = Readable.from(files)\n\n\t\treturn stream.map(\n\t\t\tasync (file: string): Promise<PieceSyncResult> => {\n\t\t\t\ttry {\n\t\t\t\t\tconst markdown = await this.get(file)\n\t\t\t\t\tconst dbPiece = await selectItem(db, markdown.filePath)\n\n\t\t\t\t\tif (dbPiece) {\n\t\t\t\t\t\tconst readStream = this._storage.createReadStream(markdown.filePath)\n\t\t\t\t\t\tconst newHash = await calculateHashFromFile(readStream)\n\t\t\t\t\t\tconst cache = await getCache(db, markdown.filePath)\n\t\t\t\t\t\tif (options?.force || cache?.content_hash !== newHash) {\n\t\t\t\t\t\t\tif (!options?.dryRun) {\n\t\t\t\t\t\t\t\tawait this.syncMarkdownUpdate(db, markdown, dbPiece)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\treturn { action: 'updated', file }\n\t\t\t\t\t\t}\n\t\t\t\t\t\t\n\t\t\t\t\t\tif (!options?.dryRun) {\n\t\t\t\t\t\t\tawait updateCache(db, markdown.filePath, newHash)\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\treturn { action: 'skipped', file }\n\t\t\t\t\t} else {\n\t\t\t\t\t\tif (!options?.dryRun) {\n\t\t\t\t\t\t\tawait this.syncMarkdownAdd(db, markdown)\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn { action: 'added', file }\n\t\t\t\t\t}\n\t\t\t\t} catch (error) {\n\t\t\t\t\treturn { file, error: true, message: `error syncing piece: ${error}` }\n\t\t\t\t}\n\t\t\t},\n\t\t\t{ concurrency: cpus().length }\n\t\t) as Readable & AsyncIterable<PieceSyncResult>\n\t}\n\n\tasync syncMarkdownAdd(db: LuzzleDatabase, markdown: PieceMarkdown<F>): Promise<void> {\n\t\tconst createInput = makePieceItemInsertable(this._pieceName, markdown, this._schema)\n\t\tconst readStream = this._storage.createReadStream(markdown.filePath)\n\t\tconst hash = await calculateHashFromFile(readStream)\n\n\t\tawait insertItem(db, createInput)\n\t\tawait addCache(db, markdown.filePath, hash)\n\t}\n\n\tasync syncMarkdown(db: LuzzleDatabase, markdown: PieceMarkdown<F>): Promise<void> {\n\t\tconst dbPiece = await selectItem(db, markdown.filePath)\n\n\t\tif (dbPiece) {\n\t\t\tawait this.syncMarkdownUpdate(db, markdown, dbPiece)\n\t\t} else {\n\t\t\tawait this.syncMarkdownAdd(db, markdown)\n\t\t}\n\t}\n\n\tasync syncMarkdownUpdate(\n\t\tdb: LuzzleDatabase,\n\t\tmarkdown: PieceMarkdown<F>,\n\t\tdata: LuzzleSelectable<'pieces_items'>\n\t): Promise<void> {\n\t\tconst updateInput = makePieceItemUpdatable(markdown, this._schema, data, false)\n\t\tawait updateItem(db, markdown.filePath, updateInput)\n\t\tconst readStream = this._storage.createReadStream(markdown.filePath)\n\t\tconst hash = await calculateHashFromFile(readStream)\n\n\t\tawait updateCache(db, data.file_path, hash)\n\t}\n\n\ttoMarkdown(data: LuzzleSelectable<'pieces_items'>): PieceMarkdown<F> {\n\t\tconst frontmatter = JSON.parse(data.frontmatter_json)\n\t\tconst frontmatterJson: Record<string, unknown> = {}\n\t\tconst dataKeys = Object.keys(frontmatter)\n\t\tconst fields = getPieceFrontmatterSchemaFields(this._schema).filter(\n\t\t\t(f): f is PieceFrontmatterSchemaField & { name: string } =>\n\t\t\t\t!!f.name && dataKeys.includes(f.name)\n\t\t)\n\n\t\tfields.forEach((field) => {\n\t\t\tconst name = field.name\n\t\t\tconst value = frontmatter[name]\n\n\t\t\tfrontmatterJson[name] = databaseValueToPieceFrontmatterValue(value, field)\n\t\t})\n\n\t\treturn makePieceMarkdown(data.file_path, data.type, data.note_markdown, frontmatterJson as F)\n\t}\n\n\tgetField(markdown: PieceMarkdown<F>, fieldPath: string): PieceFrontMatterValue | undefined {\n\t\treturn getFrontmatterValue(markdown.frontmatter, fieldPath)\n\t}\n\n\tasync setFields(\n\t\tmarkdown: PieceMarkdown<F>,\n\t\tfields: Record<string, unknown>\n\t): Promise<PieceMarkdown<F>> {\n\t\tlet updatedMarkdown = markdown\n\n\t\tfor (const fieldname in fields) {\n\t\t\tupdatedMarkdown = await this.setField(updatedMarkdown, fieldname, fields[fieldname])\n\t\t}\n\n\t\treturn updatedMarkdown\n\t}\n\n\tasync removeFields(\n\t\tmarkdown: PieceMarkdown<F>,\n\t\tfields: string[]\n\t): Promise<PieceMarkdown<Omit<F, keyof F>>> {\n\t\tlet updatedMarkdown = markdown as unknown as PieceMarkdown<Omit<F, keyof F>>\n\n\t\tfor (const field of fields) {\n\t\t\tupdatedMarkdown = (await this.removeField(\n\t\t\t\tupdatedMarkdown as unknown as PieceMarkdown<F>,\n\t\t\t\tfield\n\t\t\t)) as unknown as PieceMarkdown<Omit<F, keyof F>>\n\t\t}\n\n\t\treturn updatedMarkdown\n\t}\n\n\tasync setField(\n\t\tmarkdown: PieceMarkdown<F>,\n\t\tfieldPath: string,\n\t\tvalue: unknown\n\t): Promise<PieceMarkdown<F>> {\n\t\tconst pieceField = findFrontmatterField(this.fields, fieldPath)\n\n\t\tif (!pieceField) {\n\t\t\tthrow new Error(`${fieldPath} is not a field in ${this._pieceName} ${markdown.filePath}`)\n\t\t}\n\n\t\tconst isArray = pieceField.type === 'array'\n\t\tconst itemField = isArray ? { ...pieceField.items, name: pieceField.name } : pieceField\n\t\tconst values = Array.isArray(value) ? value : [value]\n\n\t\tconst updatedFrontmatter = structuredClone(markdown.frontmatter)\n\n\t\ttry {\n\t\t\tif (isArray) {\n\t\t\t\tconst current = getFrontmatterValue(updatedFrontmatter, fieldPath)\n\t\t\t\tif (!Array.isArray(current)) {\n\t\t\t\t\tsetFrontmatterValue(updatedFrontmatter, fieldPath, [] as unknown as PieceFrontMatterValue)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tfor (const one of values) {\n\t\t\t\tconst pieceValue = await makePieceValue(itemField, one)\n\n\t\t\t\tconst val = isAttachableStream(pieceValue)\n\t\t\t\t\t? await savePieceFieldAsset(\n\t\t\t\t\t\tmarkdown.filePath,\n\t\t\t\t\t\titemField as PieceFrontmatterSchemaField,\n\t\t\t\t\t\tpieceValue,\n\t\t\t\t\t\tthis._storage\n\t\t\t\t\t)\n\t\t\t\t\t: (pieceValue as PieceFrontMatterValue)\n\n\t\t\t\tsetFrontmatterValue(updatedFrontmatter, fieldPath, val)\n\t\t\t}\n\t\t} catch (e) {\n\t\t\tconst error = e as Error\n\t\t\tconsole.error(`could not set field ${fieldPath}: ${error.message}`)\n\t\t\treturn markdown\n\t\t}\n\n\t\treturn makePieceMarkdown(markdown.filePath, markdown.piece, markdown.note, updatedFrontmatter)\n\t}\n\n\tasync removeField(\n\t\tmarkdown: PieceMarkdown<F>,\n\t\tfieldPath: string,\n\t\tvalue?: number | string | boolean\n\t): Promise<PieceMarkdown<F>> {\n\t\tconst pieceField = findFrontmatterField(this.fields, fieldPath)\n\n\t\tif (!pieceField) {\n\t\t\tthrow new Error(`${fieldPath} is not a field in ${this._pieceName} ${markdown.filePath}`)\n\t\t}\n\n\t\tif (pieceField.nullable !== true) {\n\t\t\tthrow new Error(`${fieldPath} is a required field in ${this._pieceName} ${markdown.filePath}`)\n\t\t}\n\n\t\tconst updatedFrontmatter = structuredClone(markdown.frontmatter)\n\n\t\tif (value === undefined) {\n\t\t\tunsetFrontmatterValue(updatedFrontmatter, fieldPath)\n\t\t} else {\n\t\t\tconst pieceValue = await makePieceValue(pieceField, value)\n\t\t\tconst current = getFrontmatterValue(updatedFrontmatter, fieldPath)\n\n\t\t\tif (Array.isArray(current) && !isAttachableStream(pieceValue)) {\n\t\t\t\tconst index = current.indexOf(pieceValue as PieceFrontMatterValue)\n\t\t\t\tif (index !== -1) {\n\t\t\t\t\tunsetFrontmatterValue(updatedFrontmatter, `${fieldPath}.${index}`)\n\t\t\t\t} else {\n\t\t\t\t\treturn markdown\n\t\t\t\t}\n\t\t\t} else if (current === pieceValue) {\n\t\t\t\tunsetFrontmatterValue(updatedFrontmatter, fieldPath)\n\t\t\t}\n\t\t}\n\n\t\treturn makePieceMarkdown(markdown.filePath, markdown.piece, markdown.note, updatedFrontmatter)\n\t}\n}\n\nexport default Piece\n"]}

package/dist/src/pieces/index.d.ts

@@ -1,6 +1,7 @@
 export * from './utils/markdown.js';
 export * from './utils/frontmatter.js';
 export * from './utils/frontmatter.path.js';
+export * from './utils/piece.js';
 export * from './json.schema.js';
 export * from './items.js';
 export * from './item.js';

package/dist/src/pieces/index.js

@@ -1,6 +1,7 @@
 export * from './utils/markdown.js';
 export * from './utils/frontmatter.js';
 export * from './utils/frontmatter.path.js';
+export * from './utils/piece.js';
 export * from './json.schema.js';
 export * from './items.js';
 export * from './item.js';

package/dist/src/pieces/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/pieces/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAA;AACnC,cAAc,wBAAwB,CAAA;AACtC,cAAc,6BAA6B,CAAA;AAC3C,cAAc,kBAAkB,CAAA;AAChC,cAAc,YAAY,CAAA;AAC1B,cAAc,WAAW,CAAA;AACzB,cAAc,cAAc,CAAA;AAC5B,OAAO,EAAE,OAAO,IAAI,KAAK,EAA+C,MAAM,YAAY,CAAA;AAC1F,OAAO,EAAE,OAAO,IAAI,MAAM,EAAiD,MAAM,aAAa,CAAA","sourcesContent":["export * from './utils/markdown.js'\nexport * from './utils/frontmatter.js'\nexport * from './utils/frontmatter.path.js'\nexport * from './json.schema.js'\nexport * from './items.js'\nexport * from './item.js'\nexport * from './manager.js'\nexport { default as Piece, type PieceSyncResult, type PiecePruneResult } from './Piece.js'\nexport { default as Pieces, type PiecesSyncResult, type PiecesPruneResult } from './Pieces.js'\n"]}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/pieces/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAA;AACnC,cAAc,wBAAwB,CAAA;AACtC,cAAc,6BAA6B,CAAA;AAC3C,cAAc,kBAAkB,CAAA;AAChC,cAAc,kBAAkB,CAAA;AAChC,cAAc,YAAY,CAAA;AAC1B,cAAc,WAAW,CAAA;AACzB,cAAc,cAAc,CAAA;AAC5B,OAAO,EAAE,OAAO,IAAI,KAAK,EAA+C,MAAM,YAAY,CAAA;AAC1F,OAAO,EAAE,OAAO,IAAI,MAAM,EAAiD,MAAM,aAAa,CAAA","sourcesContent":["export * from './utils/markdown.js'\nexport * from './utils/frontmatter.js'\nexport * from './utils/frontmatter.path.js'\nexport * from './utils/piece.js'\nexport * from './json.schema.js'\nexport * from './items.js'\nexport * from './item.js'\nexport * from './manager.js'\nexport { default as Piece, type PieceSyncResult, type PiecePruneResult } from './Piece.js'\nexport { default as Pieces, type PiecesSyncResult, type PiecesPruneResult } from './Pieces.js'\n"]}

package/dist/src/pieces/utils/piece.d.ts

@@ -10,7 +10,11 @@ declare function detectStreamFileType(stream: Readable, maxBytes?: number): Prom
     type: import("file-type").FileTypeResult | undefined;
     stream: Readable;
 }>;
-declare function makePieceAttachment(file: string, field: PieceFrontmatterSchemaField, stream: AttachableStream, storage: LuzzleStorage): Promise<string>;
+declare function savePieceAsset(file: string, filename: string, stream: Readable, storage: LuzzleStorage): Promise<string>;
+declare function savePieceAsset(file: string, source: string, storage: LuzzleStorage, options?: {
+    name?: string;
+}): Promise<string>;
+declare function savePieceFieldAsset(file: string, field: PieceFrontmatterSchemaField, stream: AttachableStream, storage: LuzzleStorage): Promise<string>;
 declare function isAttachableStream(value: unknown): value is AttachableStream;
 declare function makePieceValue(field: PieceFrontmatterSchemaField, value: PieceFrontMatterValue | AttachableStream): Promise<PieceFrontMatterValue | AttachableStream>;
-export { calculateHashFromFile, isAttachableStream, makePieceAttachment, makePieceValue, detectStreamFileType, type AttachableStream, };
+export { calculateHashFromFile, isAttachableStream, savePieceFieldAsset, makePieceValue, detectStreamFileType, savePieceAsset, type AttachableStream, };

package/dist/src/pieces/utils/piece.js

@@ -100,27 +100,47 @@ async function detectStreamFileType(stream, maxBytes = 4100) {
         stream: Readable.from(gen()),
     };
 }
-async function makePieceAttachment(file, field, stream, storage) {
-    const format = field.type === 'array' ? field.items.format : field.format;
-    /* c8 ignore next 3 */
-    if (format !== 'asset') {
-        throw new Error(`${field} is not an attachable field for ${file}`);
+async function savePieceAsset(file, filenameOrSource, streamOrStorage, storageOrOptions) {
+    let finalStream;
+    let targetFilename;
+    let storage;
+    if (streamOrStorage &&
+        typeof streamOrStorage === 'object' &&
+        'pipe' in streamOrStorage &&
+        typeof streamOrStorage.pipe === 'function') {
+        targetFilename = filenameOrSource;
+        finalStream = streamOrStorage;
+        storage = storageOrOptions;
+    }
+    else {
+        storage = streamOrStorage;
+        const options = storageOrOptions;
+        const res = await downloadToStream(filenameOrSource);
+        finalStream = res.stream;
+        const originalFilename = res.filename || 'attachment';
+        if (options?.name) {
+            const ext = path.extname(originalFilename);
+            const hasExt = path.extname(options.name) !== '';
+            targetFilename = hasExt ? options.name : options.name + ext;
+        }
+        else {
+            targetFilename = originalFilename;
+        }
     }
     const pieceDir = file.replace(/\.[^.]+$/, '');
     const attachDir = path.join(ASSETS_DIRECTORY, pieceDir);
     const exists = await storage.exists(attachDir);
-    const sourceInfo = stream.filename;
     if (!exists) {
         await storage.makeDirectory(attachDir);
     }
-    const { type: detectedType, stream: finalStream } = await detectStreamFileType(stream.stream);
-    const sourceBasename = sourceInfo
-        ? path.basename(sourceInfo, path.extname(sourceInfo))
-        : field.name;
+    const { type: detectedType, stream: detectedStream } = await detectStreamFileType(finalStream);
+    const sourceBasename = targetFilename
+        ? path.basename(targetFilename, path.extname(targetFilename))
+        : 'attachment';
     const sourceExt = detectedType
         ? '.' + detectedType.ext
-        : sourceInfo
-            ? path.extname(sourceInfo)
+        : targetFilename
+            ? path.extname(targetFilename)
             : '';
     let relPath = path.join(attachDir, sourceBasename + sourceExt);
     let counter = 2;
@@ -128,9 +148,18 @@ async function makePieceAttachment(file, field, stream, storage) {
         relPath = path.join(attachDir, sourceBasename + '-' + counter + sourceExt);
         counter++;
     }
-    await pipeline(finalStream, storage.createWriteStream(relPath));
+    await pipeline(detectedStream, storage.createWriteStream(relPath));
     return relPath;
 }
+async function savePieceFieldAsset(file, field, stream, storage) {
+    const format = field.type === 'array' ? field.items.format : field.format;
+    /* c8 ignore next 3 */
+    if (format !== 'asset') {
+        throw new Error(`${field} is not an attachable field for ${file}`);
+    }
+    const filename = stream.filename || field.name;
+    return savePieceAsset(file, filename, stream.stream, storage);
+}
 function isAttachableStream(value) {
     return (typeof value === 'object' &&
         value !== null &&
@@ -163,5 +192,5 @@ async function makePieceValue(field, value) {
     }
     return value;
 }
-export { calculateHashFromFile, isAttachableStream, makePieceAttachment, makePieceValue, detectStreamFileType, };
+export { calculateHashFromFile, isAttachableStream, savePieceFieldAsset, makePieceValue, detectStreamFileType, savePieceAsset, };
 //# sourceMappingURL=piece.js.map

package/dist/src/pieces/utils/piece.js.map

@@ -1 +1 @@
-{"version":3,"file":"piece.js","sourceRoot":"","sources":["../../../../src/pieces/utils/piece.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AACnC,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAA;AACjC,OAAO,EAAE,gBAAgB,EAAE,MAAM,IAAI,CAAA;AACrC,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAA;AAC1C,OAAO,IAAI,MAAM,MAAM,CAAA;AACvB,OAAO,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAA;AAC9C,OAAO,GAAG,MAAM,KAAK,CAAA;AAGrB,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAI/C,KAAK,UAAU,gBAAgB,CAAC,SAAiB;IAChD,IAAI,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACpC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACtC,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE;gBACtC,eAAe,EAAE,KAAK;gBACtB,OAAO,EAAE;oBACR,YAAY,EAAE,gDAAgD;iBAC9D;gBACD,KAAK,EAAE;oBACN,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,CAAC,KAAK,CAAC;iBAChB;gBACD,OAAO,EAAE;oBACR,OAAO,EAAE,KAAK;iBACd;aACD,CAAC,CAAA;YACF,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBAC5B,OAAO,CAAC,KAAK,CAAC,+BAA+B,SAAS,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAA;gBACzE,MAAM,CAAC,GAAG,CAAC,CAAA;YACZ,CAAC,CAAC,CAAA;YACF,QAAQ,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,QAAQ,EAAE,EAAE;gBACpC,IAAI,QAAQ,CAAC,UAAU,IAAI,GAAG,EAAE,CAAC;oBAChC,OAAO,CAAC,KAAK,CAAC,+BAA+B,SAAS,UAAU,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAA;oBACtF,MAAM,CAAC,IAAI,KAAK,CAAC,eAAe,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAA;gBACxD,CAAC;qBAAM,CAAC;oBACP,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,CAAA;oBAC3D,OAAO,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAA;gBACxC,CAAC;YACF,CAAC,CAAC,CAAA;QACH,CAAC,CAAC,CAAA;IACH,CAAC;IAED,MAAM,IAAI,GAAG,SAAS,CAAA;IACtB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAA;IAEnD,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;QACnC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACtC,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;YACrC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBAC1B,OAAO,CAAC,KAAK,CAAC,iCAAiC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAA;gBAC7D,MAAM,CAAC,GAAG,CAAC,CAAA;YACZ,CAAC,CAAC,CAAA;YACF,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;gBACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;gBACpC,OAAO,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAA;YAC9B,CAAC,CAAC,CAAA;QACH,CAAC,CAAC,CAAA;IACH,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,sBAAsB,CAAC,CAAA;AACpD,CAAC;AAED,SAAS,qBAAqB,CAAC,MAAgB;IAC9C,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;IAE9B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACtC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YAC1B,OAAO,CAAC,KAAK,CAAC,uCAAuC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAA;YACnE,MAAM,CAAC,GAAG,CAAC,CAAA;QACZ,CAAC,CAAC,CAAA;QACF,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;QAC9C,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IACpD,CAAC,CAAC,CAAA;AACH,CAAC;AAED,KAAK,UAAU,oBAAoB,CAAC,MAAgB,EAAE,QAAQ,GAAG,IAAI;IACpE,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAA;IAC/C,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,IAAI,GAAG,KAAK,CAAA;IAEhB,OAAO,MAAM,GAAG,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;QAClC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,IAAI,GAAG,IAAI,CAAA;YACX,MAAK;QACN,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAChF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAClB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAA;IACvB,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IACpC,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC,MAAM,CAAC,CAAA;IAE7C,KAAK,SAAS,CAAC,CAAC,GAAG;QAClB,IAAI,MAAM,GAAG,CAAC;YAAE,MAAM,MAAM,CAAA;QAC5B,IAAI,IAAI;YAAE,OAAM;QAEhB,IAAI,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;QAChC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,KAAK,CAAA;YAChB,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;QAC7B,CAAC;IACF,CAAC;IAED,OAAO;QACN,IAAI;QACJ,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;KAC5B,CAAA;AACF,CAAC;AAED,KAAK,UAAU,mBAAmB,CACjC,IAAY,EACZ,KAAkC,EAClC,MAAwB,EACxB,OAAsB;IAEtB,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAA;IAEzE,sBAAsB;IACtB,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,mCAAmC,IAAI,EAAE,CAAC,CAAA;IACnE,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;IAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAA;IACvD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;IAC9C,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAA;IAElC,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,MAAM,OAAO,CAAC,aAAa,CAAC,SAAS,CAAC,CAAA;IACvC,CAAC;IAED,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IAE7F,MAAM,cAAc,GAAG,UAAU;QAChC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACrD,CAAC,CAAC,KAAK,CAAC,IAAI,CAAA;IAEb,MAAM,SAAS,GAAG,YAAY;QAC7B,CAAC,CAAC,GAAG,GAAG,YAAY,CAAC,GAAG;QACxB,CAAC,CAAC,UAAU;YACX,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC;YAC1B,CAAC,CAAC,EAAE,CAAA;IAEN,IAAI,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,GAAG,SAAS,CAAC,CAAA;IAC9D,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,OAAO,MAAM,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACtC,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,GAAG,GAAG,GAAG,OAAO,GAAG,SAAS,CAAC,CAAA;QAC1E,OAAO,EAAE,CAAA;IACV,CAAC;IAED,MAAM,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAA;IAE/D,OAAO,OAAO,CAAA;AACf,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACzC,OAAO,CACN,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,QAAQ,IAAI,KAAK;QACjB,OAAQ,KAA0B,CAAC,MAAM,EAAE,IAAI,KAAK,UAAU,CAC9D,CAAA;AACF,CAAC;AAED,KAAK,UAAU,cAAc,CAC5B,KAAkC,EAClC,KAA+C;IAE/C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,KAAK,OAAO,CAAA;IACtC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAA;IAC1D,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAA;IAEpD,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;QACxB,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC/B,IAAI,KAAK,CAAC,UAAU,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBACxC,OAAO,KAAK,CAAA;YACb,CAAC;YACD,OAAO,gBAAgB,CAAC,KAAK,CAAC,CAAA;QAC/B,CAAC;aAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5F,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,6BAA6B,CAAC,CAAA;QACvD,CAAC;aAAM,CAAC;YACP,OAAO,KAAK,CAAA;QACb,CAAC;IACF,CAAC;SAAM,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QAC/B,OAAO,YAAY,CAAC,IAAI,CAAC,KAAe,CAAC,CAAA;IAC1C,CAAC;SAAM,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QAC/B,OAAO,QAAQ,CAAC,KAAe,CAAC,CAAA;IACjC,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,OAAO,EACN,qBAAqB,EACrB,kBAAkB,EAClB,mBAAmB,EACnB,cAAc,EACd,oBAAoB,GAEpB,CAAA","sourcesContent":["import { stat } from 'fs/promises'\nimport { createHash } from 'crypto'\nimport { Readable } from 'stream'\nimport { createReadStream } from 'fs'\nimport { pipeline } from 'stream/promises'\nimport path from 'path'\nimport { fileTypeFromBuffer } from 'file-type'\nimport got from 'got'\nimport { PieceFrontmatterSchemaField, PieceFrontMatterValue } from './frontmatter.js'\nimport LuzzleStorage from '../../storage/abstract.js'\nimport { ASSETS_DIRECTORY } from '../assets.js'\n\ntype AttachableStream = { stream: Readable; filename?: string }\n\nasync function downloadToStream(fileOrUrl: string): Promise<AttachableStream> {\n\tif (/https?:\\/\\//i.test(fileOrUrl)) {\n\t\treturn new Promise((resolve, reject) => {\n\t\t\tconst download = got.stream(fileOrUrl, {\n\t\t\t\tthrowHttpErrors: false,\n\t\t\t\theaders: {\n\t\t\t\t\t'user-agent': 'luzzle/core (https://github.com/eleith/luzzle)',\n\t\t\t\t},\n\t\t\t\tretry: {\n\t\t\t\t\tlimit: 3,\n\t\t\t\t\tmethods: ['GET'],\n\t\t\t\t},\n\t\t\t\ttimeout: {\n\t\t\t\t\trequest: 10000,\n\t\t\t\t},\n\t\t\t})\n\t\t\tdownload.on('error', (err) => {\n\t\t\t\tconsole.error(`Error downloading file from ${fileOrUrl}: ${err.message}`)\n\t\t\t\treject(err)\n\t\t\t})\n\t\t\tdownload.on('response', (response) => {\n\t\t\t\tif (response.statusCode >= 400) {\n\t\t\t\t\tconsole.error(`Error downloading file from ${fileOrUrl}: http ${response.statusCode}`)\n\t\t\t\t\treject(new Error(`HTTP Error: ${response.statusCode}`))\n\t\t\t\t} else {\n\t\t\t\t\tconst filename = path.basename(new URL(fileOrUrl).pathname)\n\t\t\t\t\tresolve({ stream: download, filename })\n\t\t\t\t}\n\t\t\t})\n\t\t})\n\t}\n\n\tconst file = fileOrUrl\n\tconst fileStat = await stat(file).catch(() => null)\n\n\tif (fileStat && fileStat.isFile()) {\n\t\treturn new Promise((resolve, reject) => {\n\t\t\tconst stream = createReadStream(file)\n\t\t\tstream.on('error', (err) => {\n\t\t\t\tconsole.error(`Error reading file from path: ${err.message}`)\n\t\t\t\treject(err)\n\t\t\t})\n\t\t\tstream.on('open', () => {\n\t\t\t\tconst filename = path.basename(file)\n\t\t\t\tresolve({ stream, filename })\n\t\t\t})\n\t\t})\n\t}\n\n\tthrow new Error(`${fileOrUrl} is not a valid file`)\n}\n\nfunction calculateHashFromFile(stream: Readable): Promise<string> {\n\tconst hash = createHash('md5')\n\n\treturn new Promise((resolve, reject) => {\n\t\tstream.on('error', (err) => {\n\t\t\tconsole.error(`Error calculating hash from stream: ${err.message}`)\n\t\t\treject(err)\n\t\t})\n\t\tstream.on('data', (data) => hash.update(data))\n\t\tstream.on('end', () => resolve(hash.digest('hex')))\n\t})\n}\n\nasync function detectStreamFileType(stream: Readable, maxBytes = 4100) {\n\tconst iterator = stream[Symbol.asyncIterator]()\n\tconst chunks: Buffer[] = []\n\tlet length = 0\n\tlet done = false\n\n\twhile (length < maxBytes) {\n\t\tconst next = await iterator.next()\n\t\tif (next.done) {\n\t\t\tdone = true\n\t\t\tbreak\n\t\t}\n\t\tconst chunk = Buffer.isBuffer(next.value) ? next.value : Buffer.from(next.value)\n\t\tchunks.push(chunk)\n\t\tlength += chunk.length\n\t}\n\n\tconst buffer = Buffer.concat(chunks)\n\tconst type = await fileTypeFromBuffer(buffer)\n\n\tasync function* gen() {\n\t\tif (length > 0) yield buffer\n\t\tif (done) return\n\n\t\tlet next = await iterator.next()\n\t\twhile (!next.done) {\n\t\t\tyield next.value\n\t\t\tnext = await iterator.next()\n\t\t}\n\t}\n\n\treturn {\n\t\ttype,\n\t\tstream: Readable.from(gen()),\n\t}\n}\n\nasync function makePieceAttachment(\n\tfile: string,\n\tfield: PieceFrontmatterSchemaField,\n\tstream: AttachableStream,\n\tstorage: LuzzleStorage\n): Promise<string> {\n\tconst format = field.type === 'array' ? field.items.format : field.format\n\n\t/* c8 ignore next 3 */\n\tif (format !== 'asset') {\n\t\tthrow new Error(`${field} is not an attachable field for ${file}`)\n\t}\n\n\tconst pieceDir = file.replace(/\\.[^.]+$/, '')\n\tconst attachDir = path.join(ASSETS_DIRECTORY, pieceDir)\n\tconst exists = await storage.exists(attachDir)\n\tconst sourceInfo = stream.filename\n\n\tif (!exists) {\n\t\tawait storage.makeDirectory(attachDir)\n\t}\n\n\tconst { type: detectedType, stream: finalStream } = await detectStreamFileType(stream.stream)\n\n\tconst sourceBasename = sourceInfo\n\t\t? path.basename(sourceInfo, path.extname(sourceInfo))\n\t\t: field.name\n\n\tconst sourceExt = detectedType\n\t\t? '.' + detectedType.ext\n\t\t: sourceInfo\n\t\t\t? path.extname(sourceInfo)\n\t\t\t: ''\n\n\tlet relPath = path.join(attachDir, sourceBasename + sourceExt)\n\tlet counter = 2\n\n\twhile (await storage.exists(relPath)) {\n\t\trelPath = path.join(attachDir, sourceBasename + '-' + counter + sourceExt)\n\t\tcounter++\n\t}\n\n\tawait pipeline(finalStream, storage.createWriteStream(relPath))\n\n\treturn relPath\n}\n\nfunction isAttachableStream(value: unknown): value is AttachableStream {\n\treturn (\n\t\ttypeof value === 'object' &&\n\t\tvalue !== null &&\n\t\t'stream' in value &&\n\t\ttypeof (value as AttachableStream).stream?.pipe === 'function'\n\t)\n}\n\nasync function makePieceValue(\n\tfield: PieceFrontmatterSchemaField,\n\tvalue: PieceFrontMatterValue | AttachableStream\n): Promise<PieceFrontMatterValue | AttachableStream> {\n\tconst isArray = field.type === 'array'\n\tconst format = isArray ? field.items.format : field.format\n\tconst type = isArray ? field.items.type : field.type\n\n\tif (format === 'asset') {\n\t\tif (typeof value === 'string') {\n\t\t\tif (value.startsWith(ASSETS_DIRECTORY)) {\n\t\t\t\treturn value\n\t\t\t}\n\t\t\treturn downloadToStream(value)\n\t\t} else if (typeof value === 'number' || typeof value === 'boolean' || Array.isArray(value)) {\n\t\t\tthrow new Error(`${field} must be a string or stream`)\n\t\t} else {\n\t\t\treturn value\n\t\t}\n\t} else if (type === 'boolean') {\n\t\treturn /1|true|yes/.test(value as string)\n\t} else if (type === 'integer') {\n\t\treturn parseInt(value as string)\n\t}\n\n\treturn value\n}\n\nexport {\n\tcalculateHashFromFile,\n\tisAttachableStream,\n\tmakePieceAttachment,\n\tmakePieceValue,\n\tdetectStreamFileType,\n\ttype AttachableStream,\n}\n"]}
+{"version":3,"file":"piece.js","sourceRoot":"","sources":["../../../../src/pieces/utils/piece.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AACnC,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAA;AACjC,OAAO,EAAE,gBAAgB,EAAE,MAAM,IAAI,CAAA;AACrC,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAA;AAC1C,OAAO,IAAI,MAAM,MAAM,CAAA;AACvB,OAAO,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAA;AAC9C,OAAO,GAAG,MAAM,KAAK,CAAA;AAGrB,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAI/C,KAAK,UAAU,gBAAgB,CAAC,SAAiB;IAChD,IAAI,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACpC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACtC,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE;gBACtC,eAAe,EAAE,KAAK;gBACtB,OAAO,EAAE;oBACR,YAAY,EAAE,gDAAgD;iBAC9D;gBACD,KAAK,EAAE;oBACN,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,CAAC,KAAK,CAAC;iBAChB;gBACD,OAAO,EAAE;oBACR,OAAO,EAAE,KAAK;iBACd;aACD,CAAC,CAAA;YACF,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBAC5B,OAAO,CAAC,KAAK,CAAC,+BAA+B,SAAS,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAA;gBACzE,MAAM,CAAC,GAAG,CAAC,CAAA;YACZ,CAAC,CAAC,CAAA;YACF,QAAQ,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,QAAQ,EAAE,EAAE;gBACpC,IAAI,QAAQ,CAAC,UAAU,IAAI,GAAG,EAAE,CAAC;oBAChC,OAAO,CAAC,KAAK,CAAC,+BAA+B,SAAS,UAAU,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAA;oBACtF,MAAM,CAAC,IAAI,KAAK,CAAC,eAAe,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAA;gBACxD,CAAC;qBAAM,CAAC;oBACP,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,CAAA;oBAC3D,OAAO,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAA;gBACxC,CAAC;YACF,CAAC,CAAC,CAAA;QACH,CAAC,CAAC,CAAA;IACH,CAAC;IAED,MAAM,IAAI,GAAG,SAAS,CAAA;IACtB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAA;IAEnD,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;QACnC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACtC,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;YACrC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBAC1B,OAAO,CAAC,KAAK,CAAC,iCAAiC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAA;gBAC7D,MAAM,CAAC,GAAG,CAAC,CAAA;YACZ,CAAC,CAAC,CAAA;YACF,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;gBACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;gBACpC,OAAO,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAA;YAC9B,CAAC,CAAC,CAAA;QACH,CAAC,CAAC,CAAA;IACH,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,sBAAsB,CAAC,CAAA;AACpD,CAAC;AAED,SAAS,qBAAqB,CAAC,MAAgB;IAC9C,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;IAE9B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACtC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YAC1B,OAAO,CAAC,KAAK,CAAC,uCAAuC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAA;YACnE,MAAM,CAAC,GAAG,CAAC,CAAA;QACZ,CAAC,CAAC,CAAA;QACF,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;QAC9C,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IACpD,CAAC,CAAC,CAAA;AACH,CAAC;AAED,KAAK,UAAU,oBAAoB,CAAC,MAAgB,EAAE,QAAQ,GAAG,IAAI;IACpE,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAA;IAC/C,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,IAAI,GAAG,KAAK,CAAA;IAEhB,OAAO,MAAM,GAAG,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;QAClC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,IAAI,GAAG,IAAI,CAAA;YACX,MAAK;QACN,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAChF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAClB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAA;IACvB,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IACpC,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC,MAAM,CAAC,CAAA;IAE7C,KAAK,SAAS,CAAC,CAAC,GAAG;QAClB,IAAI,MAAM,GAAG,CAAC;YAAE,MAAM,MAAM,CAAA;QAC5B,IAAI,IAAI;YAAE,OAAM;QAEhB,IAAI,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;QAChC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,KAAK,CAAA;YAChB,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;QAC7B,CAAC;IACF,CAAC;IAED,OAAO;QACN,IAAI;QACJ,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;KAC5B,CAAA;AACF,CAAC;AAcD,KAAK,UAAU,cAAc,CAC5B,IAAY,EACZ,gBAAwB,EACxB,eAAyC,EACzC,gBAAoD;IAEpD,IAAI,WAAqB,CAAA;IACzB,IAAI,cAAsB,CAAA;IAC1B,IAAI,OAAsB,CAAA;IAE1B,IACC,eAAe;QACf,OAAO,eAAe,KAAK,QAAQ;QACnC,MAAM,IAAI,eAAe;QACzB,OAAQ,eAAuC,CAAC,IAAI,KAAK,UAAU,EAClE,CAAC;QACF,cAAc,GAAG,gBAAgB,CAAA;QACjC,WAAW,GAAG,eAA2B,CAAA;QACzC,OAAO,GAAG,gBAAiC,CAAA;IAC5C,CAAC;SAAM,CAAC;QACP,OAAO,GAAG,eAAgC,CAAA;QAC1C,MAAM,OAAO,GAAG,gBAAiD,CAAA;QACjE,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,gBAAgB,CAAC,CAAA;QACpD,WAAW,GAAG,GAAG,CAAC,MAAM,CAAA;QACxB,MAAM,gBAAgB,GAAG,GAAG,CAAC,QAAQ,IAAI,YAAY,CAAA;QACrD,IAAI,OAAO,EAAE,IAAI,EAAE,CAAC;YACnB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAA;YAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,CAAA;YAChD,cAAc,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,GAAG,GAAG,CAAA;QAC5D,CAAC;aAAM,CAAC;YACP,cAAc,GAAG,gBAAgB,CAAA;QAClC,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;IAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAA;IACvD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;IAE9C,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,MAAM,OAAO,CAAC,aAAa,CAAC,SAAS,CAAC,CAAA;IACvC,CAAC;IAED,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,oBAAoB,CAAC,WAAW,CAAC,CAAA;IAE9F,MAAM,cAAc,GAAG,cAAc;QACpC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QAC7D,CAAC,CAAC,YAAY,CAAA;IAEf,MAAM,SAAS,GAAG,YAAY;QAC7B,CAAC,CAAC,GAAG,GAAG,YAAY,CAAC,GAAG;QACxB,CAAC,CAAC,cAAc;YACf,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC;YAC9B,CAAC,CAAC,EAAE,CAAA;IAEN,IAAI,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,GAAG,SAAS,CAAC,CAAA;IAC9D,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,OAAO,MAAM,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACtC,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,GAAG,GAAG,GAAG,OAAO,GAAG,SAAS,CAAC,CAAA;QAC1E,OAAO,EAAE,CAAA;IACV,CAAC;IAED,MAAM,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAA;IAElE,OAAO,OAAO,CAAA;AACf,CAAC;AAED,KAAK,UAAU,mBAAmB,CACjC,IAAY,EACZ,KAAkC,EAClC,MAAwB,EACxB,OAAsB;IAEtB,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAA;IAEzE,sBAAsB;IACtB,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,mCAAmC,IAAI,EAAE,CAAC,CAAA;IACnE,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAA;IAC9C,OAAO,cAAc,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;AAC9D,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACzC,OAAO,CACN,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,QAAQ,IAAI,KAAK;QACjB,OAAQ,KAA0B,CAAC,MAAM,EAAE,IAAI,KAAK,UAAU,CAC9D,CAAA;AACF,CAAC;AAED,KAAK,UAAU,cAAc,CAC5B,KAAkC,EAClC,KAA+C;IAE/C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,KAAK,OAAO,CAAA;IACtC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAA;IAC1D,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAA;IAEpD,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;QACxB,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC/B,IAAI,KAAK,CAAC,UAAU,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBACxC,OAAO,KAAK,CAAA;YACb,CAAC;YACD,OAAO,gBAAgB,CAAC,KAAK,CAAC,CAAA;QAC/B,CAAC;aAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5F,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,6BAA6B,CAAC,CAAA;QACvD,CAAC;aAAM,CAAC;YACP,OAAO,KAAK,CAAA;QACb,CAAC;IACF,CAAC;SAAM,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QAC/B,OAAO,YAAY,CAAC,IAAI,CAAC,KAAe,CAAC,CAAA;IAC1C,CAAC;SAAM,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QAC/B,OAAO,QAAQ,CAAC,KAAe,CAAC,CAAA;IACjC,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,OAAO,EACN,qBAAqB,EACrB,kBAAkB,EAClB,mBAAmB,EACnB,cAAc,EACd,oBAAoB,EACpB,cAAc,GAEd,CAAA","sourcesContent":["import { stat } from 'fs/promises'\nimport { createHash } from 'crypto'\nimport { Readable } from 'stream'\nimport { createReadStream } from 'fs'\nimport { pipeline } from 'stream/promises'\nimport path from 'path'\nimport { fileTypeFromBuffer } from 'file-type'\nimport got from 'got'\nimport { PieceFrontmatterSchemaField, PieceFrontMatterValue } from './frontmatter.js'\nimport LuzzleStorage from '../../storage/abstract.js'\nimport { ASSETS_DIRECTORY } from '../assets.js'\n\ntype AttachableStream = { stream: Readable; filename?: string }\n\nasync function downloadToStream(fileOrUrl: string): Promise<AttachableStream> {\n\tif (/https?:\\/\\//i.test(fileOrUrl)) {\n\t\treturn new Promise((resolve, reject) => {\n\t\t\tconst download = got.stream(fileOrUrl, {\n\t\t\t\tthrowHttpErrors: false,\n\t\t\t\theaders: {\n\t\t\t\t\t'user-agent': 'luzzle/core (https://github.com/eleith/luzzle)',\n\t\t\t\t},\n\t\t\t\tretry: {\n\t\t\t\t\tlimit: 3,\n\t\t\t\t\tmethods: ['GET'],\n\t\t\t\t},\n\t\t\t\ttimeout: {\n\t\t\t\t\trequest: 10000,\n\t\t\t\t},\n\t\t\t})\n\t\t\tdownload.on('error', (err) => {\n\t\t\t\tconsole.error(`Error downloading file from ${fileOrUrl}: ${err.message}`)\n\t\t\t\treject(err)\n\t\t\t})\n\t\t\tdownload.on('response', (response) => {\n\t\t\t\tif (response.statusCode >= 400) {\n\t\t\t\t\tconsole.error(`Error downloading file from ${fileOrUrl}: http ${response.statusCode}`)\n\t\t\t\t\treject(new Error(`HTTP Error: ${response.statusCode}`))\n\t\t\t\t} else {\n\t\t\t\t\tconst filename = path.basename(new URL(fileOrUrl).pathname)\n\t\t\t\t\tresolve({ stream: download, filename })\n\t\t\t\t}\n\t\t\t})\n\t\t})\n\t}\n\n\tconst file = fileOrUrl\n\tconst fileStat = await stat(file).catch(() => null)\n\n\tif (fileStat && fileStat.isFile()) {\n\t\treturn new Promise((resolve, reject) => {\n\t\t\tconst stream = createReadStream(file)\n\t\t\tstream.on('error', (err) => {\n\t\t\t\tconsole.error(`Error reading file from path: ${err.message}`)\n\t\t\t\treject(err)\n\t\t\t})\n\t\t\tstream.on('open', () => {\n\t\t\t\tconst filename = path.basename(file)\n\t\t\t\tresolve({ stream, filename })\n\t\t\t})\n\t\t})\n\t}\n\n\tthrow new Error(`${fileOrUrl} is not a valid file`)\n}\n\nfunction calculateHashFromFile(stream: Readable): Promise<string> {\n\tconst hash = createHash('md5')\n\n\treturn new Promise((resolve, reject) => {\n\t\tstream.on('error', (err) => {\n\t\t\tconsole.error(`Error calculating hash from stream: ${err.message}`)\n\t\t\treject(err)\n\t\t})\n\t\tstream.on('data', (data) => hash.update(data))\n\t\tstream.on('end', () => resolve(hash.digest('hex')))\n\t})\n}\n\nasync function detectStreamFileType(stream: Readable, maxBytes = 4100) {\n\tconst iterator = stream[Symbol.asyncIterator]()\n\tconst chunks: Buffer[] = []\n\tlet length = 0\n\tlet done = false\n\n\twhile (length < maxBytes) {\n\t\tconst next = await iterator.next()\n\t\tif (next.done) {\n\t\t\tdone = true\n\t\t\tbreak\n\t\t}\n\t\tconst chunk = Buffer.isBuffer(next.value) ? next.value : Buffer.from(next.value)\n\t\tchunks.push(chunk)\n\t\tlength += chunk.length\n\t}\n\n\tconst buffer = Buffer.concat(chunks)\n\tconst type = await fileTypeFromBuffer(buffer)\n\n\tasync function* gen() {\n\t\tif (length > 0) yield buffer\n\t\tif (done) return\n\n\t\tlet next = await iterator.next()\n\t\twhile (!next.done) {\n\t\t\tyield next.value\n\t\t\tnext = await iterator.next()\n\t\t}\n\t}\n\n\treturn {\n\t\ttype,\n\t\tstream: Readable.from(gen()),\n\t}\n}\n\nasync function savePieceAsset(\n\tfile: string,\n\tfilename: string,\n\tstream: Readable,\n\tstorage: LuzzleStorage\n): Promise<string>\nasync function savePieceAsset(\n\tfile: string,\n\tsource: string,\n\tstorage: LuzzleStorage,\n\toptions?: { name?: string }\n): Promise<string>\nasync function savePieceAsset(\n\tfile: string,\n\tfilenameOrSource: string,\n\tstreamOrStorage: Readable | LuzzleStorage,\n\tstorageOrOptions?: LuzzleStorage | { name?: string }\n): Promise<string> {\n\tlet finalStream: Readable\n\tlet targetFilename: string\n\tlet storage: LuzzleStorage\n\n\tif (\n\t\tstreamOrStorage &&\n\t\ttypeof streamOrStorage === 'object' &&\n\t\t'pipe' in streamOrStorage &&\n\t\ttypeof (streamOrStorage as unknown as Readable).pipe === 'function'\n\t) {\n\t\ttargetFilename = filenameOrSource\n\t\tfinalStream = streamOrStorage as Readable\n\t\tstorage = storageOrOptions as LuzzleStorage\n\t} else {\n\t\tstorage = streamOrStorage as LuzzleStorage\n\t\tconst options = storageOrOptions as { name?: string } | undefined\n\t\tconst res = await downloadToStream(filenameOrSource)\n\t\tfinalStream = res.stream\n\t\tconst originalFilename = res.filename || 'attachment'\n\t\tif (options?.name) {\n\t\t\tconst ext = path.extname(originalFilename)\n\t\t\tconst hasExt = path.extname(options.name) !== ''\n\t\t\ttargetFilename = hasExt ? options.name : options.name + ext\n\t\t} else {\n\t\t\ttargetFilename = originalFilename\n\t\t}\n\t}\n\n\tconst pieceDir = file.replace(/\\.[^.]+$/, '')\n\tconst attachDir = path.join(ASSETS_DIRECTORY, pieceDir)\n\tconst exists = await storage.exists(attachDir)\n\n\tif (!exists) {\n\t\tawait storage.makeDirectory(attachDir)\n\t}\n\n\tconst { type: detectedType, stream: detectedStream } = await detectStreamFileType(finalStream)\n\n\tconst sourceBasename = targetFilename\n\t\t? path.basename(targetFilename, path.extname(targetFilename))\n\t\t: 'attachment'\n\n\tconst sourceExt = detectedType\n\t\t? '.' + detectedType.ext\n\t\t: targetFilename\n\t\t\t? path.extname(targetFilename)\n\t\t\t: ''\n\n\tlet relPath = path.join(attachDir, sourceBasename + sourceExt)\n\tlet counter = 2\n\n\twhile (await storage.exists(relPath)) {\n\t\trelPath = path.join(attachDir, sourceBasename + '-' + counter + sourceExt)\n\t\tcounter++\n\t}\n\n\tawait pipeline(detectedStream, storage.createWriteStream(relPath))\n\n\treturn relPath\n}\n\nasync function savePieceFieldAsset(\n\tfile: string,\n\tfield: PieceFrontmatterSchemaField,\n\tstream: AttachableStream,\n\tstorage: LuzzleStorage\n): Promise<string> {\n\tconst format = field.type === 'array' ? field.items.format : field.format\n\n\t/* c8 ignore next 3 */\n\tif (format !== 'asset') {\n\t\tthrow new Error(`${field} is not an attachable field for ${file}`)\n\t}\n\n\tconst filename = stream.filename || field.name\n\treturn savePieceAsset(file, filename, stream.stream, storage)\n}\n\nfunction isAttachableStream(value: unknown): value is AttachableStream {\n\treturn (\n\t\ttypeof value === 'object' &&\n\t\tvalue !== null &&\n\t\t'stream' in value &&\n\t\ttypeof (value as AttachableStream).stream?.pipe === 'function'\n\t)\n}\n\nasync function makePieceValue(\n\tfield: PieceFrontmatterSchemaField,\n\tvalue: PieceFrontMatterValue | AttachableStream\n): Promise<PieceFrontMatterValue | AttachableStream> {\n\tconst isArray = field.type === 'array'\n\tconst format = isArray ? field.items.format : field.format\n\tconst type = isArray ? field.items.type : field.type\n\n\tif (format === 'asset') {\n\t\tif (typeof value === 'string') {\n\t\t\tif (value.startsWith(ASSETS_DIRECTORY)) {\n\t\t\t\treturn value\n\t\t\t}\n\t\t\treturn downloadToStream(value)\n\t\t} else if (typeof value === 'number' || typeof value === 'boolean' || Array.isArray(value)) {\n\t\t\tthrow new Error(`${field} must be a string or stream`)\n\t\t} else {\n\t\t\treturn value\n\t\t}\n\t} else if (type === 'boolean') {\n\t\treturn /1|true|yes/.test(value as string)\n\t} else if (type === 'integer') {\n\t\treturn parseInt(value as string)\n\t}\n\n\treturn value\n}\n\nexport {\n\tcalculateHashFromFile,\n\tisAttachableStream,\n\tsavePieceFieldAsset,\n\tmakePieceValue,\n\tdetectStreamFileType,\n\tsavePieceAsset,\n\ttype AttachableStream,\n}\n"]}