@digipair/skill-vespa 0.90.0 → 0.91.0-0

package/.swcrc

@@ -0,0 +1,28 @@
+{
+  "jsc": {
+    "target": "es2017",
+    "parser": {
+      "syntax": "typescript",
+      "decorators": true,
+      "dynamicImport": true
+    },
+    "transform": {
+      "decoratorMetadata": true,
+      "legacyDecorator": true
+    },
+    "keepClassNames": true,
+    "externalHelpers": true,
+    "loose": true
+  },
+  "module": {
+    "type": "es6"
+  },
+  "sourceMaps": true,
+  "exclude": [
+    "jest.config.ts",
+    ".*\\.spec.tsx?$",
+    ".*\\.test.tsx?$",
+    "./src/jest-setup.ts$",
+    "./**/jest-setup.ts$"
+  ]
+}

package/README.md

@@ -0,0 +1,7 @@
+# mylib
+
+This library was generated with [Nx](https://nx.dev).
+
+## Building
+
+Run `nx build mylib` to build the library.

package/eslint.config.mjs

@@ -0,0 +1,22 @@
+import baseConfig from '../../eslint.config.mjs';
+
+export default [
+  ...baseConfig,
+  {
+    files: ['**/*.json'],
+    rules: {
+      '@nx/dependency-checks': [
+        'error',
+        {
+          ignoredFiles: [
+            '{projectRoot}/eslint.config.{js,cjs,mjs}',
+            '{projectRoot}/rollup.config.{js,ts,mjs,mts,cjs,cts}',
+          ],
+        },
+      ],
+    },
+    languageOptions: {
+      parser: await import('jsonc-eslint-parser'),
+    },
+  },
+];

package/package.json

@@ -1,12 +1,28 @@
 {
   "name": "@digipair/skill-vespa",
-  "version": "0.90.0",
+  "version": "0.91.0-0",
+  "type": "module",
+  "main": "dist/libs/skill-vespa/index.cjs.js",
+  "module": "dist/libs/skill-vespa/index.esm.js",
+  "types": "dist/libs/skill-vespa/index.esm.d.ts",
+  "exports": {
+    "./package.json": "./libs/skill-vespa/package.json",
+    ".": {
+      "development": "./dist/libs/skill-vespa/src/index.ts",
+      "types": "./dist/libs/skill-vespa/index.esm.d.ts",
+      "import": "./dist/libs/skill-vespa/index.esm.js",
+      "default": "./dist/libs/skill-vespa/index.cjs.js"
+    }
+  },
   "keywords": [
     "digipair",
     "service",
     "tool"
   ],
-  "dependencies": {},
-  "main": "./index.cjs.js",
-  "module": "./index.esm.js"
-}
+  "nx": {
+    "name": "skill-vespa"
+  },
+  "dependencies": {
+    "@digipair/engine": "0.91.0-0"
+  }
+}

package/rollup.config.cjs

@@ -0,0 +1,28 @@
+const { withNx } = require('@nx/rollup/with-nx');
+
+module.exports = withNx(
+  {
+    main: 'libs/skill-vespa/src/index.ts',
+    outputPath: 'dist/libs/skill-vespa',
+    tsConfig: 'libs/skill-vespa/tsconfig.lib.json',
+    compiler: 'swc',
+    format: ['esm', "cjs"],
+    assets: [
+      {
+        input: 'libs/skill-vespa/',
+        glob: 'package.json',
+        output: '.'
+      },
+      {
+        input: 'libs/skill-vespa/src/',
+        glob: '*.json',
+        output: '.'
+      }
+    ]
+  },
+  {
+    // Provide additional rollup configuration here. See: https://rollupjs.org/configuration-options
+    // e.g.
+    // output: { sourcemap: true },
+  }
+);

package/src/handlebars.d.ts

@@ -0,0 +1 @@
+declare module 'handlebars/dist/handlebars.min.js';

package/src/lib/skill-vespa.spec.ts

@@ -0,0 +1,7 @@
+import { skillVespa } from './skill-vespa';
+
+describe('skillVespa', () => {
+  it('should work', () => {
+    expect(skillVespa()).toEqual('skill-vespa');
+  });
+});

package/src/lib/skill-vespa.ts

@@ -0,0 +1,389 @@
+/* eslint-disable @typescript-eslint/no-unused-vars */
+import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
+import { v4 } from 'uuid';
+import { PinsSettings, executePinsList } from '@digipair/engine';
+
+const VESPA_SERVER = process.env['VESPA_SERVER'] ?? 'http://localhost:8080';
+
+class VespaService {
+  private async searchDocuments(
+    baseUrl: string,
+    collection: string,
+    query: string,
+    options: any = {},
+    signal: AbortSignal,
+  ): Promise<any[]> {
+    const response = await fetch(`${baseUrl}/search/`, {
+      signal,
+      method: 'POST',
+      body: JSON.stringify({
+        yql: `select * from ${collection} where ${query}`,
+        ...options,
+      }),
+      headers: {
+        'Content-Type': 'application/json',
+      },
+    });
+
+    if (!response.ok) {
+      const error = await response.json();
+      console.error('Error - VespaService:getDocuments - fetching', error.root?.errors ?? error);
+      throw new Error(`Error - VespaService:getDocuments - fetching ${collection}`);
+    }
+
+    const messages = ((await response.json()).root.children || []).map((child: any) =>
+      child.id === 'group:root:0' ? child.children[0].children : child.fields,
+    );
+    return messages;
+  }
+
+  private async searchParentDocuments(
+    baseUrl: string,
+    collection: string,
+    query: string,
+    options: any = {},
+    signal: AbortSignal,
+  ): Promise<any[]> {
+    const documents = [];
+    const uuids = [];
+    const queryUuids = [];
+    const chunks = await this.searchDocuments(baseUrl, collection, query, options, signal);
+
+    documents.push(...chunks.filter(({ is_parent }) => is_parent));
+    uuids.push(...chunks.filter(({ is_parent }) => is_parent).map(({ uuid }) => uuid));
+    for (const chunk of chunks) {
+      if (uuids.indexOf(chunk.parent_uuid) < 0 && queryUuids.indexOf(chunk.parent_uuid) < 0) {
+        queryUuids.push(chunk.parent_uuid);
+      }
+    }
+
+    if (queryUuids.length > 0) {
+      documents.push(
+        ...(await this.searchDocuments(
+          baseUrl,
+          collection,
+          `uuid in (${queryUuids.map(uuid => `"${uuid}"`).join(',')})`,
+          options,
+          signal,
+        )),
+      );
+    }
+
+    const results = [] as any[];
+    for (const chunk of chunks) {
+      const document = documents.find(({ parent_uuid }) => parent_uuid === chunk.parent_uuid);
+      if (document && !results.find(({ uuid }) => uuid === document.uuid)) {
+        results.push({ ...document, matchfeatures: chunk.matchfeatures });
+      }
+    }
+
+    return results;
+  }
+
+  private async prepareDocuments(documents: any[]) {
+    const parents = documents.map(document => {
+      const uuid = v4();
+      return { ...document, uuid, parent_uuid: uuid, is_parent: true };
+    });
+
+    const chunksPromises = parents.map(async parent => {
+      const text = parent.content;
+      const v128 = new RecursiveCharacterTextSplitter({
+        chunkSize: 128,
+        chunkOverlap: 12,
+      });
+      const v256 = new RecursiveCharacterTextSplitter({
+        chunkSize: 256,
+        chunkOverlap: 25,
+      });
+      const v512 = new RecursiveCharacterTextSplitter({
+        chunkSize: 512,
+        chunkOverlap: 51,
+      });
+      const output = [
+        ...(text.length > 128 ? await v128.createDocuments([text]) : []),
+        ...(text.length > 256 ? await v256.createDocuments([text]) : []),
+        ...(text.length > 512 ? await v512.createDocuments([text]) : []),
+      ];
+
+      return output.map(({ pageContent }) => ({
+        ...parent,
+        content: pageContent,
+        uuid: v4(),
+        is_parent: false,
+      }));
+    });
+
+    const chunks = await Promise.all(chunksPromises);
+    const result = [...parents, ...chunks.flat()];
+
+    return result;
+  }
+
+  private async pushDocuments(
+    modelEmbeddings: any,
+    baseUrl: string,
+    namespace: string,
+    collection: string,
+    documents: any[],
+    signal: AbortSignal,
+    asynchronous: boolean,
+  ) {
+    const results = [];
+
+    for (const document of documents) {
+      // eslint-disable-next-line no-control-regex
+      const content = document.content.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '');
+      const content_embedding = await modelEmbeddings.embedQuery(content);
+
+      const response = await fetch(
+        `${baseUrl}/document/v1/${namespace}/${collection}/docid/${document.uuid}`,
+        {
+          signal,
+          method: 'POST',
+          body: JSON.stringify(
+            asynchronous
+              ? { ...document, content }
+              : {
+                  ...document,
+                  content,
+                  content_embedding: await modelEmbeddings.embedQuery(content),
+                },
+          ),
+          headers: {
+            'Content-Type': 'application/json',
+          },
+        },
+      );
+
+      const json = await response.json();
+      if (!json.id) {
+        throw new Error(
+          `Error - VespaService:pushDocuments - pushing ${collection} - ${json.message}`,
+        );
+      }
+      results.push(json);
+
+      if (asynchronous) {
+        setImmediate(async () => {
+          const content_embedding = await modelEmbeddings.embedQuery(content);
+          await fetch(`${baseUrl}/document/v1/${namespace}/${collection}/docid/${document.uuid}`, {
+            method: 'POST',
+            body: JSON.stringify({
+              fields: { ...document, content, content_embedding },
+            }),
+            headers: {
+              'Content-Type': 'application/json',
+            },
+          });
+        });
+      }
+    }
+
+    return results;
+  }
+
+  async find(params: any, _pinsSettingsList: PinsSettings[], context: any): Promise<any> {
+    const {
+      baseUrl = context.privates.VESPA_SERVER ?? VESPA_SERVER,
+      collection = 'knowledge',
+      limit = 100,
+      orderby = '',
+      query,
+      grouping,
+    } = params;
+
+    if (
+      orderby !== '' &&
+      !/^([a-zA-Z0-9_-]+)(,\s*[a-zA-Z0-9_-]+)*(\s+(asc|desc))?$/.test(orderby)
+    ) {
+      throw new Error('vespa:find - Invalid orderby parameter');
+    }
+
+    const orderbySecured = orderby === '' ? '' : `order by ${orderby}`;
+    const results = await this.searchDocuments(
+      baseUrl,
+      collection,
+      `is_parent = true and userQuery() ${orderbySecured} limit ${parseInt(limit)}` +
+        (grouping ? ` | ${grouping}` : ''),
+      {
+        query,
+      },
+      context.protected?.signal,
+    );
+
+    return results;
+  }
+
+  async search(params: any, _pinsSettingsList: PinsSettings[], context: any): Promise<any> {
+    const {
+      embeddings = context.privates.MODEL_EMBEDDINGS,
+      baseUrl = context.privates.VESPA_SERVER ?? VESPA_SERVER,
+      collection = 'knowledge',
+      limit = 100,
+      orderby = '',
+      targetHits = 50,
+      filter = 'true',
+      language,
+      query,
+    } = params;
+
+    if (
+      orderby !== '' &&
+      !/^([a-zA-Z0-9_-]+)(,\s*[a-zA-Z0-9_-]+)*(\s+(asc|desc))?$/.test(orderby)
+    ) {
+      throw new Error('vespa:find - Invalid orderby parameter');
+    }
+
+    const orderbySecured = orderby === '' ? '' : `order by ${orderby}`;
+    const modelEmbeddings = await executePinsList(
+      embeddings,
+      context,
+      `${context.__PATH__}.embeddings`,
+    );
+    const queryEmbedding = await modelEmbeddings.embedQuery(query);
+    const results = await this.searchParentDocuments(
+      baseUrl,
+      collection,
+      `((userQuery()) or ({targetHits:${targetHits}}nearestNeighbor(content_embedding,q))) and (${filter}) ${orderbySecured} limit ${parseInt(
+        limit,
+      )}`,
+      {
+        'ranking.profile': 'fusion',
+        'input.query(q)': queryEmbedding,
+        query,
+        language,
+      },
+      context.protected?.signal,
+    );
+
+    return results;
+  }
+
+  async textSplitter(params: any, _pinsSettingsList: PinsSettings[], _context: any): Promise<any> {
+    const { size = 1024, overlap = 102, source, text } = params;
+    const splitter = new RecursiveCharacterTextSplitter({
+      chunkSize: size,
+      chunkOverlap: overlap,
+    });
+
+    return (await splitter.createDocuments([text])).map(({ pageContent }) => ({
+      source,
+      content: pageContent,
+    }));
+  }
+
+  async push(params: any, _pinsSettingsList: PinsSettings[], context: any): Promise<any> {
+    const {
+      embeddings = context.privates.MODEL_EMBEDDINGS,
+      baseUrl = context.privates.VESPA_SERVER ?? VESPA_SERVER,
+      namespace = context.privates.VESPA_NAMESPACE ??
+        process.env['VESPA_NAMESPACE'] ??
+        'Digipair_default',
+      collection = 'knowledge',
+      asynchronous = false,
+      documents,
+    } = params;
+    const modelEmbeddings = await executePinsList(
+      embeddings,
+      context,
+      `${context.__PATH__}.embeddings`,
+    );
+    const results = await this.prepareDocuments(documents);
+
+    await this.pushDocuments(
+      modelEmbeddings,
+      baseUrl,
+      namespace,
+      collection,
+      results,
+      context.protected?.signal,
+      asynchronous,
+    );
+
+    return results.filter(({ is_parent }) => is_parent).map(({ uuid }) => uuid);
+  }
+
+  async remove(params: any, _pinsSettingsList: PinsSettings[], context: any): Promise<any> {
+    const {
+      baseUrl = context.privates.VESPA_SERVER ?? VESPA_SERVER,
+      namespace = context.privates.VESPA_NAMESPACE ??
+        process.env['VESPA_NAMESPACE'] ??
+        'Digipair_default',
+      collection = 'knowledge',
+      selection,
+    } = params;
+    const response = await fetch(
+      `${baseUrl}/document/v1/${namespace}/${collection}/docid?selection=${encodeURI(
+        selection,
+      )}&cluster=${namespace}`,
+      {
+        signal: context.protected?.signal,
+        method: 'DELETE',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+      },
+    );
+
+    if (!response.ok) {
+      const error = await response.json();
+      console.error('Error - VespaService:remove - fetching', error.root?.errors ?? error);
+      throw new Error(`Error - VespaService:remove - fetching ${collection}`);
+    }
+
+    return await response.json();
+  }
+
+  async update(params: any, _pinsSettingsList: PinsSettings[], context: any): Promise<any> {
+    const {
+      baseUrl = context.privates.VESPA_SERVER ?? VESPA_SERVER,
+      namespace = context.privates.VESPA_NAMESPACE ??
+        process.env['VESPA_NAMESPACE'] ??
+        'Digipair_default',
+      collection,
+      id,
+      fields,
+    } = params;
+
+    let response = await fetch(`${baseUrl}/document/v1/${namespace}/${collection}/docid/${id}`, {
+      signal: context.protected.signal,
+      method: 'GET',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+    });
+    const document = (await response.json()).fields;
+
+    response = await fetch(`${baseUrl}/document/v1/${namespace}/${collection}/docid/${id}`, {
+      signal: context.protected.signal,
+      method: 'POST',
+      body: JSON.stringify({
+        fields: { ...document, ...fields },
+      }),
+      headers: {
+        'Content-Type': 'application/json',
+      },
+    });
+
+    return await response.json();
+  }
+}
+
+export const find = (params: any, pinsSettingsList: PinsSettings[], context: any) =>
+  new VespaService().find(params, pinsSettingsList, context);
+
+export const search = (params: any, pinsSettingsList: PinsSettings[], context: any) =>
+  new VespaService().search(params, pinsSettingsList, context);
+
+export const textSplitter = (params: any, pinsSettingsList: PinsSettings[], context: any) =>
+  new VespaService().textSplitter(params, pinsSettingsList, context);
+
+export const push = (params: any, pinsSettingsList: PinsSettings[], context: any) =>
+  new VespaService().push(params, pinsSettingsList, context);
+
+export const remove = (params: any, pinsSettingsList: PinsSettings[], context: any) =>
+  new VespaService().remove(params, pinsSettingsList, context);
+
+export const update = (params: any, pinsSettingsList: PinsSettings[], context: any) =>
+  new VespaService().update(params, pinsSettingsList, context);

package/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "files": [],
+  "include": [],
+  "references": [
+    {
+      "path": "../engine"
+    },
+    {
+      "path": "./tsconfig.lib.json"
+    }
+  ]
+}

package/tsconfig.lib.json

@@ -0,0 +1,19 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "dist",
+    "tsBuildInfoFile": "dist/tsconfig.lib.tsbuildinfo",
+    "emitDeclarationOnly": true,
+    "module": "esnext",
+    "moduleResolution": "node",
+    "forceConsistentCasingInFileNames": true,
+    "types": ["node"]
+  },
+  "include": ["src/**/*.ts"],
+  "references": [
+    {
+      "path": "../engine/tsconfig.lib.json"
+    }
+  ]
+}

package/index.cjs.d.ts

@@ -1 +0,0 @@
-export * from "./src/index";