@gmickel/gno 1.8.0 → 1.10.0

package/src/ingestion/frontmatter.ts

@@ -117,6 +117,168 @@ function parseTagsValue(
   return [];
 }
 
+function parseInlineArrayValue(value: string): string[] | undefined {
+  const inlineMatch = INLINE_ARRAY_REGEX.exec(value.trim());
+  if (!inlineMatch) {
+    return undefined;
+  }
+  const rawItems = inlineMatch[1];
+  if (rawItems === undefined) {
+    return [];
+  }
+
+  return rawItems
+    .split(",")
+    .map((item) => unquoteScalar(item.trim()))
+    .filter((item) => item.length > 0);
+}
+
+function unquoteScalar(value: string): string {
+  const trimmed = value.trim();
+  if (trimmed.length < 2) {
+    return trimmed;
+  }
+  const first = trimmed[0];
+  const last = trimmed.at(-1);
+  if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
+    return trimmed.slice(1, -1).trim();
+  }
+  return trimmed;
+}
+
+function parseBlockArrayValue(lines: string[], startIdx: number): string[] {
+  const items: string[] = [];
+  for (let i = startIdx + 1; i < lines.length; i++) {
+    const line = lines[i];
+    if (line === undefined) break;
+    const itemMatch = YAML_ARRAY_ITEM_REGEX.exec(line);
+    if (itemMatch?.[1]) {
+      const item = unquoteScalar(itemMatch[1]);
+      if (item.length > 0) {
+        items.push(item);
+      }
+      continue;
+    }
+    if (
+      line.trim().length > 0 &&
+      !line.startsWith(" ") &&
+      !line.startsWith("\t")
+    ) {
+      break;
+    }
+  }
+  return items;
+}
+
+function countIndent(line: string): number {
+  let indent = 0;
+  for (const char of line) {
+    if (char === " ") {
+      indent += 1;
+    } else if (char === "\t") {
+      indent += 2;
+    } else {
+      break;
+    }
+  }
+  return indent;
+}
+
+function parseNestedMapValue(
+  lines: string[],
+  startIdx: number
+): Record<string, string[]> {
+  const result: Record<string, string[]> = {};
+
+  for (let i = startIdx + 1; i < lines.length; i++) {
+    const line = lines[i];
+    if (line === undefined) break;
+    if (line.trim().length === 0) continue;
+    if (!line.startsWith(" ") && !line.startsWith("\t")) {
+      break;
+    }
+
+    const keyIndent = countIndent(line);
+    const trimmed = line.trim();
+    const colonIdx = trimmed.indexOf(":");
+    if (colonIdx <= 0) {
+      continue;
+    }
+
+    const key = trimmed.slice(0, colonIdx).trim();
+    const rawValue = trimmed.slice(colonIdx + 1).trim();
+    const values: string[] = [];
+
+    const inlineArray = parseInlineArrayValue(rawValue);
+    if (inlineArray) {
+      values.push(...inlineArray);
+    } else if (rawValue.length > 0) {
+      values.push(unquoteScalar(rawValue));
+    } else {
+      for (let j = i + 1; j < lines.length; j++) {
+        const childLine = lines[j];
+        if (childLine === undefined) break;
+        if (childLine.trim().length === 0) continue;
+        const childIndent = countIndent(childLine);
+        if (childIndent <= keyIndent) {
+          break;
+        }
+        const itemMatch = YAML_ARRAY_ITEM_REGEX.exec(childLine);
+        if (itemMatch?.[1]) {
+          const item = unquoteScalar(itemMatch[1]);
+          if (item.length > 0) {
+            values.push(item);
+          }
+        }
+      }
+    }
+
+    if (key.length > 0 && values.length > 0) {
+      result[key] = values;
+    }
+  }
+
+  return result;
+}
+
+function startsNestedMap(lines: string[], startIdx: number): boolean {
+  for (let i = startIdx + 1; i < lines.length; i++) {
+    const line = lines[i];
+    if (line === undefined) break;
+    if (line.trim().length === 0) continue;
+    if (!line.startsWith(" ") && !line.startsWith("\t")) {
+      return false;
+    }
+    return line.trim().indexOf(":") > 0;
+  }
+  return false;
+}
+
+function parseMetadataValue(
+  value: string,
+  lines: string[],
+  startIdx: number
+): string | string[] | Record<string, string[]> | undefined {
+  const trimmed = value.trim();
+  const inlineArray = parseInlineArrayValue(trimmed);
+  if (inlineArray) {
+    return inlineArray;
+  }
+
+  if (trimmed.length === 0) {
+    if (startsNestedMap(lines, startIdx)) {
+      const nestedMap = parseNestedMapValue(lines, startIdx);
+      return Object.keys(nestedMap).length > 0 ? nestedMap : undefined;
+    }
+    const blockArray = parseBlockArrayValue(lines, startIdx);
+    if (blockArray.length > 0) {
+      return blockArray;
+    }
+  }
+
+  return unquoteScalar(trimmed);
+}
+
 /**
  * Parse frontmatter from markdown source.
  * Returns extracted tags and metadata.
@@ -150,6 +312,9 @@ export function parseFrontmatter(source: string): FrontmatterResult {
     for (let i = 0; i < lines.length; i++) {
       const line = lines[i];
       if (line === undefined) continue;
+      if (line.startsWith(" ") || line.startsWith("\t")) {
+        continue;
+      }
 
       // Logseq format: tags:: value
       const logseqMatch = LOGSEQ_TAGS_REGEX.exec(line);
@@ -170,8 +335,11 @@ export function parseFrontmatter(source: string): FrontmatterResult {
       if (colonIdx > 0) {
         const key = line.slice(0, colonIdx).trim();
         const value = line.slice(colonIdx + 1).trim();
-        if (key !== "tags" && value.length > 0) {
-          result.metadata[key] = value;
+        if (key !== "tags") {
+          const metadataValue = parseMetadataValue(value, lines, i);
+          if (metadataValue !== undefined) {
+            result.metadata[key] = metadataValue;
+          }
         }
       }
     }

package/src/ingestion/index.ts

@@ -10,12 +10,14 @@ export { defaultChunker, MarkdownChunker } from "./chunker";
 export { defaultLanguageDetector, SimpleLanguageDetector } from "./language";
 // Sync service
 export { defaultSyncService, SyncService } from "./sync";
+export { resolveContentTypeRules, withContentTypeRules } from "./sync-options";
 // Types
 export type {
   ChunkerPort,
   ChunkOutput,
   ChunkParams,
   CollectionSyncResult,
+  ContentTypeSource,
   FileSyncResult,
   FileSyncStatus,
   LanguageDetectorPort,

package/src/ingestion/sync-options.ts

@@ -0,0 +1,29 @@
+/**
+ * Sync option helpers.
+ *
+ * @module src/ingestion/sync-options
+ */
+
+import type { Config, NormalizedContentTypeRule } from "../config";
+import type { SyncOptions } from "./types";
+
+import { fingerprintContentTypeRules, normalizeContentTypes } from "../config";
+
+export function resolveContentTypeRules(
+  config?: Pick<Config, "contentTypes">
+): NormalizedContentTypeRule[] {
+  return normalizeContentTypes(config?.contentTypes ?? []).rules;
+}
+
+export function withContentTypeRules(
+  options: SyncOptions = {},
+  config?: Pick<Config, "contentTypes">
+): SyncOptions {
+  const rules = options.contentTypeRules ?? resolveContentTypeRules(config);
+  return {
+    ...options,
+    contentTypeRules: rules,
+    contentTypeRulesFingerprint:
+      options.contentTypeRulesFingerprint ?? fingerprintContentTypeRules(rules),
+  };
+}