pageindex-core 1.0.0

package/dist/index.cjs

@@ -0,0 +1,139 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  MemoryVectorStore: () => MemoryVectorStore,
+  PageSplitter: () => PageSplitter,
+  createPageIndex: () => createPageIndex
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/core/PageIndex.ts
+var import_crypto = require("crypto");
+
+// src/utils/withRetry.ts
+async function withRetry(fn, retries = 3) {
+  let lastError;
+  for (let i = 0; i < retries; i++) {
+    try {
+      return await fn();
+    } catch (err) {
+      lastError = err;
+    }
+  }
+  throw lastError;
+}
+
+// src/core/PageIndex.ts
+var PageIndex = class {
+  constructor(config) {
+    this.config = config;
+    this.pages = [];
+    this.isBuilt = false;
+    if (!config.embeddingModel || !config.vectorStore) {
+      throw new Error("embeddingModel and vectorStore are required");
+    }
+  }
+  async ingest(pages) {
+    this.pages = pages.map((content, i) => ({
+      id: (0, import_crypto.randomUUID)(),
+      pageNumber: i + 1,
+      content
+    }));
+    this.isBuilt = false;
+  }
+  async build() {
+    const batchSize = this.config.batchSize ?? 5;
+    for (let i = 0; i < this.pages.length; i += batchSize) {
+      const batch = this.pages.slice(i, i + batchSize);
+      await Promise.all(
+        batch.map(async (page) => {
+          if (this.config.summarizer) {
+            page.summary = await this.config.summarizer(page.content);
+          }
+          page.embedding = await withRetry(
+            () => this.config.embeddingModel(
+              page.summary || page.content
+            )
+          );
+          await this.config.vectorStore.upsert(page);
+          this.config.logger?.("page_indexed", { id: page.id });
+        })
+      );
+    }
+    this.isBuilt = true;
+  }
+  async retrieve(query, topK = 3) {
+    if (!this.isBuilt) throw new Error("Index not built");
+    const queryEmbedding = await this.config.embeddingModel(query);
+    return this.config.vectorStore.search(queryEmbedding, {
+      topK
+    });
+  }
+};
+
+// src/public/createPageIndex.ts
+function createPageIndex(config) {
+  return new PageIndex(config);
+}
+
+// src/adapters/MemoryVectorStore.ts
+var MemoryVectorStore = class {
+  constructor() {
+    this.store = /* @__PURE__ */ new Map();
+  }
+  async upsert(page) {
+    this.store.set(page.id, page);
+  }
+  async search(queryEmbedding, options) {
+    const scored = Array.from(this.store.values()).map((page) => ({
+      page,
+      score: cosineSimilarity(queryEmbedding, page.embedding || [])
+    }));
+    return scored.sort((a, b) => b.score - a.score).slice(0, options.topK).map((s) => s.page);
+  }
+};
+function cosineSimilarity(a, b) {
+  let dot = 0, normA = 0, normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * (b[i] || 0);
+    normA += a[i] ** 2;
+    normB += (b[i] || 0) ** 2;
+  }
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB) || 1);
+}
+
+// src/processors/PageSplitter.ts
+var PageSplitter = class {
+  static splitByLength(text, maxChars = 3e3) {
+    const pages = [];
+    for (let i = 0; i < text.length; i += maxChars) {
+      pages.push(text.slice(i, i + maxChars));
+    }
+    return pages;
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  MemoryVectorStore,
+  PageSplitter,
+  createPageIndex
+});

package/dist/index.d.cts

@@ -0,0 +1,49 @@
+interface PageUnit {
+    id: string;
+    pageNumber: number;
+    content: string;
+    summary?: string;
+    metadata?: Record<string, any>;
+    embedding?: number[];
+}
+interface SearchOptions {
+    topK: number;
+    filter?: Record<string, any>;
+}
+interface VectorAdapter {
+    upsert(page: PageUnit): Promise<void>;
+    search(queryEmbedding: number[], options: SearchOptions): Promise<PageUnit[]>;
+    get?(id: string): Promise<PageUnit | null>;
+    delete?(id: string): Promise<void>;
+}
+interface PageIndexConfig {
+    embeddingModel: (text: string) => Promise<number[]>;
+    summarizer?: (text: string) => Promise<string>;
+    vectorStore: VectorAdapter;
+    batchSize?: number;
+    logger?: (event: string, payload?: any) => void;
+}
+
+declare class PageIndex {
+    private config;
+    private pages;
+    private isBuilt;
+    constructor(config: PageIndexConfig);
+    ingest(pages: string[]): Promise<void>;
+    build(): Promise<void>;
+    retrieve(query: string, topK?: number): Promise<PageUnit[]>;
+}
+
+declare function createPageIndex(config: PageIndexConfig): PageIndex;
+
+declare class MemoryVectorStore implements VectorAdapter {
+    private store;
+    upsert(page: PageUnit): Promise<void>;
+    search(queryEmbedding: number[], options: SearchOptions): Promise<PageUnit[]>;
+}
+
+declare class PageSplitter {
+    static splitByLength(text: string, maxChars?: number): string[];
+}
+
+export { MemoryVectorStore, type PageIndexConfig, PageSplitter, type PageUnit, type SearchOptions, type VectorAdapter, createPageIndex };

package/dist/index.d.ts

@@ -0,0 +1,49 @@
+interface PageUnit {
+    id: string;
+    pageNumber: number;
+    content: string;
+    summary?: string;
+    metadata?: Record<string, any>;
+    embedding?: number[];
+}
+interface SearchOptions {
+    topK: number;
+    filter?: Record<string, any>;
+}
+interface VectorAdapter {
+    upsert(page: PageUnit): Promise<void>;
+    search(queryEmbedding: number[], options: SearchOptions): Promise<PageUnit[]>;
+    get?(id: string): Promise<PageUnit | null>;
+    delete?(id: string): Promise<void>;
+}
+interface PageIndexConfig {
+    embeddingModel: (text: string) => Promise<number[]>;
+    summarizer?: (text: string) => Promise<string>;
+    vectorStore: VectorAdapter;
+    batchSize?: number;
+    logger?: (event: string, payload?: any) => void;
+}
+
+declare class PageIndex {
+    private config;
+    private pages;
+    private isBuilt;
+    constructor(config: PageIndexConfig);
+    ingest(pages: string[]): Promise<void>;
+    build(): Promise<void>;
+    retrieve(query: string, topK?: number): Promise<PageUnit[]>;
+}
+
+declare function createPageIndex(config: PageIndexConfig): PageIndex;
+
+declare class MemoryVectorStore implements VectorAdapter {
+    private store;
+    upsert(page: PageUnit): Promise<void>;
+    search(queryEmbedding: number[], options: SearchOptions): Promise<PageUnit[]>;
+}
+
+declare class PageSplitter {
+    static splitByLength(text: string, maxChars?: number): string[];
+}
+
+export { MemoryVectorStore, type PageIndexConfig, PageSplitter, type PageUnit, type SearchOptions, type VectorAdapter, createPageIndex };

package/dist/index.js

@@ -0,0 +1,110 @@
+// src/core/PageIndex.ts
+import { randomUUID } from "crypto";
+
+// src/utils/withRetry.ts
+async function withRetry(fn, retries = 3) {
+  let lastError;
+  for (let i = 0; i < retries; i++) {
+    try {
+      return await fn();
+    } catch (err) {
+      lastError = err;
+    }
+  }
+  throw lastError;
+}
+
+// src/core/PageIndex.ts
+var PageIndex = class {
+  constructor(config) {
+    this.config = config;
+    this.pages = [];
+    this.isBuilt = false;
+    if (!config.embeddingModel || !config.vectorStore) {
+      throw new Error("embeddingModel and vectorStore are required");
+    }
+  }
+  async ingest(pages) {
+    this.pages = pages.map((content, i) => ({
+      id: randomUUID(),
+      pageNumber: i + 1,
+      content
+    }));
+    this.isBuilt = false;
+  }
+  async build() {
+    const batchSize = this.config.batchSize ?? 5;
+    for (let i = 0; i < this.pages.length; i += batchSize) {
+      const batch = this.pages.slice(i, i + batchSize);
+      await Promise.all(
+        batch.map(async (page) => {
+          if (this.config.summarizer) {
+            page.summary = await this.config.summarizer(page.content);
+          }
+          page.embedding = await withRetry(
+            () => this.config.embeddingModel(
+              page.summary || page.content
+            )
+          );
+          await this.config.vectorStore.upsert(page);
+          this.config.logger?.("page_indexed", { id: page.id });
+        })
+      );
+    }
+    this.isBuilt = true;
+  }
+  async retrieve(query, topK = 3) {
+    if (!this.isBuilt) throw new Error("Index not built");
+    const queryEmbedding = await this.config.embeddingModel(query);
+    return this.config.vectorStore.search(queryEmbedding, {
+      topK
+    });
+  }
+};
+
+// src/public/createPageIndex.ts
+function createPageIndex(config) {
+  return new PageIndex(config);
+}
+
+// src/adapters/MemoryVectorStore.ts
+var MemoryVectorStore = class {
+  constructor() {
+    this.store = /* @__PURE__ */ new Map();
+  }
+  async upsert(page) {
+    this.store.set(page.id, page);
+  }
+  async search(queryEmbedding, options) {
+    const scored = Array.from(this.store.values()).map((page) => ({
+      page,
+      score: cosineSimilarity(queryEmbedding, page.embedding || [])
+    }));
+    return scored.sort((a, b) => b.score - a.score).slice(0, options.topK).map((s) => s.page);
+  }
+};
+function cosineSimilarity(a, b) {
+  let dot = 0, normA = 0, normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * (b[i] || 0);
+    normA += a[i] ** 2;
+    normB += (b[i] || 0) ** 2;
+  }
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB) || 1);
+}
+
+// src/processors/PageSplitter.ts
+var PageSplitter = class {
+  static splitByLength(text, maxChars = 3e3) {
+    const pages = [];
+    for (let i = 0; i < text.length; i += maxChars) {
+      pages.push(text.slice(i, i + maxChars));
+    }
+    return pages;
+  }
+};
+export {
+  MemoryVectorStore,
+  PageSplitter,
+  createPageIndex
+};

package/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "pageindex-core",
+  "version": "1.0.0",
+  "description": "Structure-aware page-level retrieval engine (RAG alternative)",
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "sideEffects": false,
+  "scripts": {
+    "build": "tsup src/index.ts --format esm,cjs --dts",
+    "dev": "tsup src/index.ts --watch",
+    "test": "vitest",
+    "prepublishOnly": "npm run build"
+  },
+  "keywords": [
+    "rag",
+    "retrieval",
+    "llm",
+    "vector",
+    "semantic-search",
+    "pageindex"
+  ],
+  "author": "Your Name",
+  "license": "MIT"
+}

package/readme.md

@@ -0,0 +1,20 @@
+# @pageindex/core
+
+Structure-aware page-level retrieval engine for LLM systems.
+
+## Why?
+
+Traditional RAG uses arbitrary chunking which:
+- Breaks structural context
+- Loses page-level semantics
+- Causes retrieval noise
+
+PageIndex retrieves full logical pages first,
+then allows refinement — preserving structure.
+
+---
+
+## Installation
+
+```bash
+npm install @pageindex/core

package/src/__tests__/pageindex.test.ts

@@ -0,0 +1,20 @@
+import { createPageIndex } from "../index";
+import { MemoryVectorStore } from "../adapters/MemoryVectorStore";
+import { test, expect } from "vitest";
+
+test("indexes and retrieves", async () => {
+  const embedding = async (text: string) =>
+    Array(10).fill(text.length);
+
+  const index = createPageIndex({
+    embeddingModel: embedding,
+    vectorStore: new MemoryVectorStore()
+  });
+
+  await index.ingest(["hello world", "another page"]);
+  await index.build();
+
+  const results = await index.retrieve("hello");
+
+  expect(results.length).toBeGreaterThan(0);
+});

package/src/adapters/MemoryVectorStore.ts

@@ -0,0 +1,41 @@
+import {
+    VectorAdapter,
+    PageUnit,
+    SearchOptions
+  } from "../types/index.types";
+  
+  export class MemoryVectorStore implements VectorAdapter {
+    private store = new Map<string, PageUnit>();
+  
+    async upsert(page: PageUnit): Promise<void> {
+      this.store.set(page.id, page);
+    }
+  
+    async search(
+      queryEmbedding: number[],
+      options: SearchOptions
+    ): Promise<PageUnit[]> {
+  
+      const scored = Array.from(this.store.values()).map(page => ({
+        page,
+        score: cosineSimilarity(queryEmbedding, page.embedding || [])
+      }));
+  
+      return scored
+        .sort((a, b) => b.score - a.score)
+        .slice(0, options.topK)
+        .map(s => s.page);
+    }
+  }
+  
+  function cosineSimilarity(a: number[], b: number[]) {
+    let dot = 0, normA = 0, normB = 0;
+  
+    for (let i = 0; i < a.length; i++) {
+      dot += a[i] * (b[i] || 0);
+      normA += a[i] ** 2;
+      normB += (b[i] || 0) ** 2;
+    }
+  
+    return dot / (Math.sqrt(normA) * Math.sqrt(normB) || 1);
+  }

package/src/adapters/VectorAdapter.ts

@@ -0,0 +1 @@
+export * from "../types/index.types";

package/src/core/PageIndex.ts

@@ -0,0 +1,63 @@
+import { randomUUID } from "crypto";
+import {
+  PageIndexConfig,
+  PageUnit
+} from "../types/index.types";
+import { withRetry } from "../utils/withRetry";
+
+export class PageIndex {
+  private pages: PageUnit[] = [];
+  private isBuilt = false;
+
+  constructor(private config: PageIndexConfig) {
+    if (!config.embeddingModel || !config.vectorStore) {
+      throw new Error("embeddingModel and vectorStore are required");
+    }
+  }
+
+  async ingest(pages: string[]) {
+    this.pages = pages.map((content, i) => ({
+      id: randomUUID(),
+      pageNumber: i + 1,
+      content
+    }));
+    this.isBuilt = false;
+  }
+
+  async build() {
+    const batchSize = this.config.batchSize ?? 5;
+
+    for (let i = 0; i < this.pages.length; i += batchSize) {
+      const batch = this.pages.slice(i, i + batchSize);
+
+      await Promise.all(
+        batch.map(async page => {
+          if (this.config.summarizer) {
+            page.summary = await this.config.summarizer(page.content);
+          }
+
+          page.embedding = await withRetry(() =>
+            this.config.embeddingModel(
+              page.summary || page.content
+            )
+          );
+
+          await this.config.vectorStore.upsert(page);
+          this.config.logger?.("page_indexed", { id: page.id });
+        })
+      );
+    }
+
+    this.isBuilt = true;
+  }
+
+  async retrieve(query: string, topK = 3) {
+    if (!this.isBuilt) throw new Error("Index not built");
+
+    const queryEmbedding = await this.config.embeddingModel(query);
+
+    return this.config.vectorStore.search(queryEmbedding, {
+      topK
+    });
+  }
+}

package/src/index.ts

@@ -0,0 +1,4 @@
+export { createPageIndex } from "./public/createPageIndex";
+export * from "./types/index.types";
+export * from "./adapters/MemoryVectorStore";
+export * from "./processors/PageSplitter";

package/src/processors/PageSplitter.ts

@@ -0,0 +1,9 @@
+export class PageSplitter {
+    static splitByLength(text: string, maxChars = 3000) {
+      const pages = [];
+      for (let i = 0; i < text.length; i += maxChars) {
+        pages.push(text.slice(i, i + maxChars));
+      }
+      return pages;
+    }
+  }

package/src/public/createPageIndex.ts

@@ -0,0 +1,6 @@
+import { PageIndex } from "../core/PageIndex";
+import { PageIndexConfig } from "../types/index.types";
+
+export function createPageIndex(config: PageIndexConfig) {
+  return new PageIndex(config);
+}

package/src/types/index.types.ts

@@ -0,0 +1,31 @@
+export interface PageUnit {
+    id: string;
+    pageNumber: number;
+    content: string;
+    summary?: string;
+    metadata?: Record<string, any>;
+    embedding?: number[];
+  }
+  
+  export interface SearchOptions {
+    topK: number;
+    filter?: Record<string, any>;
+  }
+  
+  export interface VectorAdapter {
+    upsert(page: PageUnit): Promise<void>;
+    search(
+      queryEmbedding: number[],
+      options: SearchOptions
+    ): Promise<PageUnit[]>;
+    get?(id: string): Promise<PageUnit | null>;
+    delete?(id: string): Promise<void>;
+  }
+  
+  export interface PageIndexConfig {
+    embeddingModel: (text: string) => Promise<number[]>;
+    summarizer?: (text: string) => Promise<string>;
+    vectorStore: VectorAdapter;
+    batchSize?: number;
+    logger?: (event: string, payload?: any) => void;
+  }

package/src/utils/withRetry.ts

@@ -0,0 +1,16 @@
+export async function withRetry<T>(
+    fn: () => Promise<T>,
+    retries = 3
+  ): Promise<T> {
+    let lastError;
+  
+    for (let i = 0; i < retries; i++) {
+      try {
+        return await fn();
+      } catch (err) {
+        lastError = err;
+      }
+    }
+  
+    throw lastError;
+  }

package/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "ESNext",
+    "declaration": true,
+    "outDir": "dist",
+    "strict": true,
+    "esModuleInterop": true,
+    "moduleResolution": "Node",
+    "skipLibCheck": true
+  },
+  "include": ["src"]
+}

package/vitest.config.ts

@@ -0,0 +1,7 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    environment: "node"
+  }
+});