@remnic/core 9.27.0 → 9.29.0

package/dist/{chunk-7RZJWZ46.js → chunk-LEPOP46W.js}

@@ -1,3 +1,7 @@
+import {
+  rebuildConversationChunksFailOpen,
+  upsertConversationChunksFailOpen
+} from "./chunk-72A4L5JJ.js";
 import {
   searchConversationIndex,
   searchConversationIndexFaissFailOpen
@@ -5,10 +9,6 @@ import {
 import {
   failOpenFaissHealth
 } from "./chunk-ET356KSB.js";
-import {
-  rebuildConversationChunksFailOpen,
-  upsertConversationChunksFailOpen
-} from "./chunk-72A4L5JJ.js";
 
 // src/conversation-index/backend.ts
 function createConversationIndexBackend(options) {
@@ -235,4 +235,4 @@ function createFaissBackend(getFaiss) {
 export {
   createConversationIndexBackend
 };
-//# sourceMappingURL=chunk-7RZJWZ46.js.map
+//# sourceMappingURL=chunk-LEPOP46W.js.map

package/dist/{chunk-EYJSW2RV.js → chunk-NBDCZWP4.js}

@@ -1,9 +1,9 @@
 import {
   namespaceCollectionName
-} from "./chunk-SK3EYPKC.js";
+} from "./chunk-D2ANTDGO.js";
 import {
   NamespaceStorageRouter
-} from "./chunk-MEBQFQ26.js";
+} from "./chunk-XEQ7OT7U.js";
 import {
   namespaceIdentityFromToken,
   namespaceIdentityToken
@@ -13,7 +13,7 @@ import {
 } from "./chunk-U3PN77QT.js";
 import {
   resolveNamespaceCapabilities
-} from "./chunk-RDQGLJRR.js";
+} from "./chunk-2XRE6FLM.js";
 import {
   ALL_CATEGORY_DIRS
 } from "./chunk-VS2IYZRU.js";
@@ -207,4 +207,4 @@ export {
   verifyNamespaces,
   runNamespaceMigration
 };
-//# sourceMappingURL=chunk-EYJSW2RV.js.map
+//# sourceMappingURL=chunk-NBDCZWP4.js.map

package/dist/{chunk-XKGPIWGI.js → chunk-OFHFH655.js}

@@ -1,10 +1,13 @@
+import {
+  withHeldFileLock
+} from "./chunk-WL5EBRDH.js";
 import {
   ensureContainedSpillDir,
   listContainedSpillFiles
 } from "./chunk-GCASGYIO.js";
 import {
-  withHeldFileLock
-} from "./chunk-WL5EBRDH.js";
+  displayErrorDetail
+} from "./chunk-RL6NEVXT.js";
 import {
   MEMORY_LIFECYCLE_LEDGER_APPEND_LOCK_MAX_WAIT_MS,
   MEMORY_LIFECYCLE_LEDGER_LOCK_STALE_MS,
@@ -12,9 +15,6 @@ import {
   memoryLifecycleLedgerLockPath,
   sortMemoryLifecycleEvents
 } from "./chunk-ILGC6JH6.js";
-import {
-  displayErrorDetail
-} from "./chunk-RL6NEVXT.js";
 import {
   MAGIC_HEADER_SIZE,
   SecureStoreLockedError,
@@ -445,4 +445,4 @@ export {
   drainPendingLifecycleLedgerForSync,
   drainPendingLifecycleForSyncOrThrow
 };
-//# sourceMappingURL=chunk-XKGPIWGI.js.map
+//# sourceMappingURL=chunk-OFHFH655.js.map

package/dist/{chunk-FGBJBYWP.js → chunk-OVRQLAYR.js}

@@ -8,7 +8,7 @@ import {
 } from "./chunk-WLZYGLJ4.js";
 import {
   LocalLlmClient
-} from "./chunk-P2OZKKYP.js";
+} from "./chunk-PHWUFPCO.js";
 import {
   sessionStoragePaths
 } from "./chunk-S4DDLTPX.js";
@@ -23,7 +23,7 @@ import {
 import {
   resolveLocalLlmCapabilities,
   resolvePipelineProcessingCapabilities
-} from "./chunk-RDQGLJRR.js";
+} from "./chunk-2XRE6FLM.js";
 import {
   encodeStoragePathSegment,
   resolveSafeStoragePath
@@ -690,4 +690,4 @@ ${truncatedConversation}`;
 export {
   HourlySummarizer
 };
-//# sourceMappingURL=chunk-FGBJBYWP.js.map
+//# sourceMappingURL=chunk-OVRQLAYR.js.map

package/dist/{chunk-BXLOS5AJ.js → chunk-OWHERGF2.js}

@@ -3,7 +3,7 @@ import {
 } from "./chunk-765K3SAT.js";
 import {
   encryptCapsuleFile
-} from "./chunk-DQEMWVMT.js";
+} from "./chunk-UVYI6VIX.js";
 import {
   computeTransferOutputRel,
   isTransferPathExcluded
@@ -109,4 +109,4 @@ async function enforceRetention(outDirAbs, retentionDays) {
 export {
   backupMemoryDir
 };
-//# sourceMappingURL=chunk-BXLOS5AJ.js.map
+//# sourceMappingURL=chunk-OWHERGF2.js.map

package/dist/chunk-P2BM76KT.js

@@ -0,0 +1,528 @@
+// src/extraction-source-grounding-helpers.ts
+function isInterrogativeSourceSentence(sentence) {
+  const normalized = sentence.trim();
+  const embeddedQuestionVerbIndex = normalized.search(
+    /\b(?:determine|know|wonder|ask(?:ed)?|question|decide|check|confirm|find|figure|understand|unclear)\b/iu
+  );
+  const embeddedQuestionMarkerIndex = normalized.search(/\b(?:whether|if)\b/iu);
+  const hasEmbeddedQuestion = embeddedQuestionVerbIndex >= 0 && embeddedQuestionMarkerIndex > embeddedQuestionVerbIndex;
+  return normalized.endsWith("?") || hasEmbeddedQuestion || !normalized.includes(":") && /^(?:suppose|assuming|assume|maybe|perhaps|hypothetically|if|whether|imagine|imagining|presume|presuming|supposing|is|are|am|was|were|do|does|did|can|could|will|would|should|has|have|had|what|which|when|where|why|how|who)\b/iu.test(
+    normalized
+  );
+}
+function containsContiguousGroundingTokens(candidateTokens, sourceTokens) {
+  if (candidateTokens.length === 0 || candidateTokens.length > sourceTokens.length) return false;
+  return sourceTokens.some(
+    (_, start) => candidateTokens.every((token, offset) => sourceTokens[start + offset] === token)
+  );
+}
+function hasExplicitRoleSubjectToken(candidateRole, sourceTokensBySentence) {
+  return candidateRole !== void 0 && sourceTokensBySentence.some((tokens) => tokens.includes(candidateRole));
+}
+function splitGroundingClauses(sentences) {
+  return sentences.flatMap(
+    (sentence) => sentence.split(/,\s+(?=[\p{L}][\p{L}\p{N}'’-]*\s+\S)/gu).flatMap(
+      (commaClause) => commaClause.split(/\s*(?::|[—–])\s+|\s+\/\s+/u).flatMap(
+        (delimiterClause) => delimiterClause.split(/\s+(?:and|but|or|while|although|because)\s+/gu).map((clause) => clause.trim()).filter((clause) => clause.length > 0)
+      )
+    )
+  );
+}
+var GROUNDING_STOPWORDS = {
+  a: true,
+  an: true,
+  and: true,
+  are: true,
+  as: true,
+  at: true,
+  be: true,
+  been: true,
+  being: true,
+  but: true,
+  by: true,
+  can: true,
+  could: true,
+  did: true,
+  do: true,
+  does: true,
+  for: true,
+  from: true,
+  had: true,
+  has: true,
+  have: true,
+  he: true,
+  her: true,
+  here: true,
+  hers: true,
+  him: true,
+  his: true,
+  how: true,
+  i: true,
+  if: true,
+  in: true,
+  into: true,
+  is: true,
+  it: true,
+  its: true,
+  just: true,
+  me: true,
+  my: true,
+  no: true,
+  not: true,
+  of: true,
+  on: true,
+  only: true,
+  or: true,
+  our: true,
+  ours: true,
+  she: true,
+  so: true,
+  than: true,
+  that: true,
+  the: true,
+  their: true,
+  them: true,
+  then: true,
+  there: true,
+  these: true,
+  they: true,
+  this: true,
+  those: true,
+  to: true,
+  too: true,
+  under: true,
+  up: true,
+  us: true,
+  very: true,
+  was: true,
+  we: true,
+  were: true,
+  what: true,
+  when: true,
+  why: true,
+  where: true,
+  which: true,
+  who: true,
+  will: true,
+  with: true,
+  would: true,
+  you: true,
+  your: true
+};
+var GROUNDING_NEGATION_TOKENS = /* @__PURE__ */ new Set([
+  "no",
+  "not",
+  "false",
+  "untrue",
+  "incorrect",
+  "wrong",
+  "never",
+  "without",
+  "neither",
+  "nor",
+  "none",
+  "cannot",
+  "cant",
+  "can't",
+  "don't",
+  "doesn't",
+  "didn't",
+  "isn't",
+  "aren't",
+  "wasn't",
+  "weren't",
+  "won't",
+  "wouldn't",
+  "couldn't",
+  "shouldn't",
+  "haven't",
+  "hasn't",
+  "hadn't"
+]);
+var GROUNDING_AUXILIARY_TOKENS = /* @__PURE__ */ new Set([
+  "am",
+  "are",
+  "be",
+  "been",
+  "being",
+  "can",
+  "could",
+  "did",
+  "do",
+  "does",
+  "had",
+  "has",
+  "have",
+  "is",
+  "was",
+  "were",
+  "will",
+  "would",
+  "should"
+]);
+var GROUNDING_COPULAR_FORMS = /* @__PURE__ */ new Set([
+  "am",
+  "are",
+  "be",
+  "been",
+  "being",
+  "is",
+  "was",
+  "were"
+]);
+var GROUNDING_SUBJECT_PRONOUNS = /* @__PURE__ */ new Set([
+  "he",
+  "she",
+  "they",
+  "it",
+  "i",
+  "we",
+  "you"
+]);
+var GROUNDING_ENTITY_TYPE_PREFIXES = /* @__PURE__ */ new Set([
+  "person",
+  "project",
+  "tool",
+  "company",
+  "place",
+  "other"
+]);
+var GROUNDING_COMMON_VERB_FORMS = /* @__PURE__ */ new Set([
+  "contains",
+  "employs",
+  "gives",
+  "has",
+  "hosts",
+  "likes",
+  "makes",
+  "needs",
+  "owns",
+  "prefers",
+  "requires",
+  "runs",
+  "shares",
+  "supports",
+  "takes",
+  "uses",
+  "wants",
+  "works"
+]);
+function groundingLexemes(text) {
+  return text.normalize("NFKC").match(
+    /[\p{L}\p{N}]+(?:['’][\p{L}\p{N}]+)?(?:\+\+[\p{L}\p{N}]*|#[\p{L}\p{N}]*)?/gu
+  )?.map((rawToken) => {
+    const token = rawToken.replaceAll("\u2019", "'").toLocaleLowerCase();
+    return {
+      token,
+      preserveTerminalS: /^\p{Lu}/u.test(rawToken) && !GROUNDING_COMMON_VERB_FORMS.has(token)
+    };
+  }) ?? [];
+}
+function tokenSequence(text) {
+  return groundingLexemes(text).map(({ token }) => token);
+}
+function containsExactTokenSequence(candidate, source) {
+  const candidateTokens = tokenSequence(candidate);
+  const sourceTokens = tokenSequence(source);
+  if (candidateTokens.length === 0 || candidateTokens.length > sourceTokens.length) return false;
+  return sourceTokens.some(
+    (_, index) => candidateTokens.every((token, offset) => sourceTokens[index + offset] === token)
+  );
+}
+function isNegationCue(token) {
+  return GROUNDING_NEGATION_TOKENS.has(token) || token.endsWith("n't");
+}
+function isAttachedNegatedAuxiliary(token) {
+  return token.endsWith("n't") || token === "cannot" || token === "cant" || token === "can't";
+}
+function isNegatedAt(tokens, index) {
+  if (tokens[index - 1] === "only" && tokens[index - 2] === "not") return false;
+  const previousStart = Math.max(0, index - 2);
+  for (let i = previousStart; i < index; i += 1) {
+    if (isNegationCue(tokens[i])) return true;
+  }
+  if (tokens[index - 1] === "than" && tokens[index - 2] === "rather") return true;
+  if (tokens[index - 1] === "of" && tokens[index - 2] === "instead") return true;
+  const next = tokens[index + 1];
+  if (next !== void 0 && isAttachedNegatedAuxiliary(next)) return true;
+  const nextNext = tokens[index + 2];
+  return next !== void 0 && nextNext !== void 0 && GROUNDING_AUXILIARY_TOKENS.has(next) && isNegationCue(nextNext);
+}
+var GROUNDING_MIN_SHARED_TOKENS = 2;
+var GROUNDING_MIN_COVERAGE = 0.5;
+function stemToken(token, preserveTerminalS = false) {
+  if (token.endsWith("'s")) return token.slice(0, -2);
+  if (preserveTerminalS) return token;
+  if (token.length > 5 && token.endsWith("ing")) {
+    const stem = token.slice(0, -3);
+    return /(.)\1$/u.test(stem) ? stem.slice(0, -1) : stem;
+  }
+  if (token.length > 4 && token.endsWith("ed")) return token.slice(0, -2);
+  if (!preserveTerminalS && token.length > 3 && token.endsWith("s") && !token.endsWith("ss") && GROUNDING_COMMON_VERB_FORMS.has(token)) {
+    return token.slice(0, -1);
+  }
+  return token;
+}
+function tokenize(text) {
+  const tokens = /* @__PURE__ */ new Set();
+  for (const { token, preserveTerminalS } of groundingLexemes(text)) {
+    if (GROUNDING_STOPWORDS[token] !== true) tokens.add(stemToken(token, preserveTerminalS));
+  }
+  return tokens;
+}
+function normalizeForExactMatch(text) {
+  return text.normalize("NFKC").toLocaleLowerCase().replace(/\s+/gu, " ").trim();
+}
+function sourceSentences(source) {
+  const sentences = [];
+  let sentenceStart = 0;
+  for (let index = 0; index < source.length; index += 1) {
+    const character = source[index];
+    const nextCharacter = source[index + 1];
+    const boundary = character === "!" || character === "?" || character === ";" || character === "\n" || character === "." && (nextCharacter === void 0 || /\s/u.test(nextCharacter) || /["')\]}]/u.test(nextCharacter));
+    if (!boundary) continue;
+    let sentenceEnd = character === "\n" ? index : index + 1;
+    while (/[.!?;\n]/u.test(source[sentenceEnd] ?? "")) sentenceEnd += 1;
+    const sentence = source.slice(sentenceStart, sentenceEnd).trim();
+    if (sentence.length > 0) sentences.push(sentence);
+    sentenceStart = sentenceEnd;
+    index = sentenceEnd - 1;
+  }
+  const trailingSentence = source.slice(sentenceStart).trim();
+  if (trailingSentence.length > 0) sentences.push(trailingSentence);
+  return sentences.length > 0 ? sentences : [source];
+}
+function groundingTokenSequence(text) {
+  let lexemes = groundingLexemes(text);
+  while (lexemes.length > 0 && GROUNDING_AUXILIARY_TOKENS.has(lexemes[0].token)) {
+    lexemes = lexemes.slice(1);
+  }
+  return lexemes.filter(({ token }) => GROUNDING_STOPWORDS[token] !== true).map(({ token, preserveTerminalS }) => stemToken(token, preserveTerminalS));
+}
+function normalizedGroundingAlignmentTokenSequence(text) {
+  const normalized = normalizedGroundingTokenSequence(text);
+  const firstLexeme = groundingLexemes(text)[0];
+  if (firstLexeme === void 0 || !GROUNDING_SUBJECT_PRONOUNS.has(firstLexeme.token)) {
+    return normalized;
+  }
+  return [stemToken(firstLexeme.token, firstLexeme.preserveTerminalS), ...normalized];
+}
+function normalizedGroundingTokenSequence(text) {
+  return groundingTokenSequence(text).map((token) => token.replace(/'$/u, ""));
+}
+function areGroundingTokensCompatible(left, right) {
+  return left === right || `${left}e` === right || left === `${right}e`;
+}
+var GROUNDING_DENIAL_REPORTING_VERBS = /* @__PURE__ */ new Set([
+  "deny",
+  "denied",
+  "denies",
+  "denying",
+  "dispute",
+  "disputed",
+  "disputes",
+  "disputing",
+  "reject",
+  "rejected",
+  "rejects",
+  "rejecting",
+  "refute",
+  "refuted",
+  "refutes",
+  "refuting",
+  "contradict",
+  "contradicted",
+  "contradicts",
+  "contradicting",
+  "disprove",
+  "disproved",
+  "disproves",
+  "disproving"
+]);
+function isDenialReportingVerb(token) {
+  return GROUNDING_DENIAL_REPORTING_VERBS.has(token);
+}
+var GROUNDING_NON_ASSERTIVE_REPORTING_VERBS = /* @__PURE__ */ new Set([
+  "allege",
+  "alleged",
+  "alleges",
+  "alleging",
+  "allegedly",
+  "assert",
+  "asserted",
+  "asserts",
+  "asserting",
+  "believe",
+  "believed",
+  "believes",
+  "believing",
+  "claim",
+  "claimed",
+  "claims",
+  "claiming",
+  "doubt",
+  "doubted",
+  "doubts",
+  "doubting",
+  "uncertain",
+  "unsure",
+  "unclear",
+  "unknown",
+  "possibly",
+  "probably",
+  "imagine",
+  "imagined",
+  "imagines",
+  "imagining",
+  "report",
+  "reported",
+  "reports",
+  "reporting",
+  "reportedly",
+  "say",
+  "said",
+  "says",
+  "saying",
+  "speculate",
+  "speculated",
+  "speculates",
+  "speculating",
+  "suspect",
+  "suspected",
+  "suspects",
+  "suspecting",
+  "suppose",
+  "supposed",
+  "supposes",
+  "supposing",
+  "supposedly",
+  "think",
+  "thought",
+  "thinks",
+  "thinking"
+]);
+function isNonAssertiveReportingVerb(token) {
+  return GROUNDING_NON_ASSERTIVE_REPORTING_VERBS.has(token);
+}
+function isGroundingClauseBoundary(token) {
+  return token === "and" || token === "but" || token === "or" || token === "while" || token === "although" || token === "because";
+}
+function isNonAssertiveReportingAt(tokens, index) {
+  let boundary = 0;
+  for (let cursor = 0; cursor < index; cursor += 1) {
+    if (isGroundingClauseBoundary(tokens[cursor])) boundary = cursor + 1;
+  }
+  for (let cursor = index - 1; cursor >= boundary; cursor -= 1) {
+    if (tokens[cursor] === "that" && cursor > boundary && isNonAssertiveReportingVerb(tokens[cursor - 1])) {
+      return true;
+    }
+    if (isNonAssertiveReportingVerb(tokens[cursor]) && index - cursor <= 5) return true;
+  }
+  return false;
+}
+function isDeniedAt(tokens, index) {
+  let boundary = 0;
+  for (let cursor = 0; cursor < index; cursor += 1) {
+    if (isGroundingClauseBoundary(tokens[cursor])) boundary = cursor + 1;
+  }
+  for (let cursor = index - 1; cursor >= boundary; cursor -= 1) {
+    const token = tokens[cursor];
+    if (token === "that" && cursor > boundary && isDenialReportingVerb(tokens[cursor - 1])) {
+      return true;
+    }
+    if (isDenialReportingVerb(token) && index - cursor <= 5) return true;
+  }
+  return false;
+}
+function hasContradictoryPolarity(candidate, source) {
+  const candidateLexemes = groundingLexemes(candidate);
+  const candidateTokens = candidateLexemes.map(({ token }) => token);
+  let coherentContradiction = false;
+  for (const clause of sourceSentences(source).flatMap((sentence) => sentence.split(/\s+(?:and|but|or|while|although|because)\s+/gu))) {
+    const sourceLexemes2 = groundingLexemes(clause);
+    const sourceTokens2 = sourceLexemes2.map(({ token }) => token);
+    let sourceCursor = 0;
+    let allTokensFound = true;
+    let contradiction = false;
+    for (const [candidateIndex, candidateToken] of candidateTokens.entries()) {
+      if (GROUNDING_STOPWORDS[candidateToken] === true) continue;
+      const sourceIndex = sourceTokens2.findIndex(
+        (sourceToken, index) => index >= sourceCursor && stemToken(sourceToken, sourceLexemes2[index]?.preserveTerminalS === true) === stemToken(candidateToken, candidateLexemes[candidateIndex]?.preserveTerminalS === true)
+      );
+      if (sourceIndex === -1) {
+        allTokensFound = false;
+        break;
+      }
+      const candidateNegated = isNegatedAt(candidateTokens, candidateIndex) || isDeniedAt(candidateTokens, candidateIndex);
+      const sourceNegated = isNegatedAt(sourceTokens2, sourceIndex) || isDeniedAt(sourceTokens2, sourceIndex);
+      const candidateNonAssertive = isNonAssertiveReportingAt(candidateTokens, candidateIndex);
+      const sourceNonAssertive = isNonAssertiveReportingAt(sourceTokens2, sourceIndex);
+      if (candidateNegated !== sourceNegated || candidateNonAssertive !== sourceNonAssertive) {
+        contradiction = true;
+      }
+      sourceCursor = sourceIndex + 1;
+    }
+    if (allTokensFound) {
+      if (!contradiction) return false;
+      coherentContradiction = true;
+    }
+  }
+  if (coherentContradiction) return true;
+  const sourceLexemes = groundingLexemes(source);
+  const sourceTokens = sourceLexemes.map(({ token }) => token);
+  const sourcePositions = /* @__PURE__ */ new Map();
+  sourceLexemes.forEach(({ token, preserveTerminalS }, index) => {
+    const key = stemToken(token, preserveTerminalS);
+    const positions = sourcePositions.get(key);
+    if (positions) positions.push(index);
+    else sourcePositions.set(key, [index]);
+  });
+  for (const [candidateIndex, token] of candidateTokens.entries()) {
+    if (GROUNDING_STOPWORDS[token] === true) continue;
+    const positions = sourcePositions.get(
+      stemToken(token, candidateLexemes[candidateIndex]?.preserveTerminalS === true)
+    );
+    if (!positions || positions.length === 0) continue;
+    const sourceIndex = positions[positions.length - 1];
+    const candidateNegated = isNegatedAt(candidateTokens, candidateIndex) || isDeniedAt(candidateTokens, candidateIndex);
+    const sourceNegated = isNegatedAt(sourceTokens, sourceIndex) || isDeniedAt(sourceTokens, sourceIndex);
+    const candidateNonAssertive = isNonAssertiveReportingAt(candidateTokens, candidateIndex);
+    const sourceNonAssertive = isNonAssertiveReportingAt(sourceTokens, sourceIndex);
+    if (candidateNegated !== sourceNegated || candidateNonAssertive !== sourceNonAssertive) {
+      return true;
+    }
+  }
+  return false;
+}
+
+export {
+  isInterrogativeSourceSentence,
+  containsContiguousGroundingTokens,
+  hasExplicitRoleSubjectToken,
+  splitGroundingClauses,
+  GROUNDING_STOPWORDS,
+  GROUNDING_NEGATION_TOKENS,
+  GROUNDING_AUXILIARY_TOKENS,
+  GROUNDING_COPULAR_FORMS,
+  GROUNDING_SUBJECT_PRONOUNS,
+  GROUNDING_ENTITY_TYPE_PREFIXES,
+  GROUNDING_COMMON_VERB_FORMS,
+  groundingLexemes,
+  tokenSequence,
+  containsExactTokenSequence,
+  isNegationCue,
+  isAttachedNegatedAuxiliary,
+  isNegatedAt,
+  GROUNDING_MIN_SHARED_TOKENS,
+  GROUNDING_MIN_COVERAGE,
+  stemToken,
+  tokenize,
+  normalizeForExactMatch,
+  sourceSentences,
+  groundingTokenSequence,
+  normalizedGroundingAlignmentTokenSequence,
+  normalizedGroundingTokenSequence,
+  areGroundingTokensCompatible,
+  hasContradictoryPolarity
+};
+//# sourceMappingURL=chunk-P2BM76KT.js.map