@learncard/network-brain-service 4.0.4 → 4.1.1

package/src/config/skillEmbedding.ts

@@ -0,0 +1,202 @@
+/**
+ * LC-2206: skill-matching embeddings run on Amazon Nova 2 Multimodal Embeddings (Bedrock).
+ *
+ * Values mirror AI Passport's reviewed Nova identity (`nova2-text-1024-index-retrieval-*`):
+ * the in-Region base model ID (Nova 2 embeddings has no geo/global inference profile),
+ * us-east-1, 1024 dimensions, GENERIC_INDEX for stored skills, TEXT_RETRIEVAL for queries,
+ * NONE truncation and client-side unit normalisation.
+ *
+ * Every (model, dimension, scheme) generation gets its own Neo4j property, key property and
+ * vector index, so vectors from different models or dimensions are never mixed in one index.
+ */
+
+export const NOVA2_EMBEDDING_MODEL_ID = 'amazon.nova-2-multimodal-embeddings-v1:0';
+export const DEFAULT_SKILL_EMBEDDING_REGION = 'us-east-1';
+export const DEFAULT_SKILL_EMBEDDING_DIMENSION = 1024;
+export const DEFAULT_SKILL_EMBEDDING_TIMEOUT_MS = 15_000;
+
+/** Output sizes the Nova 2 embeddings API accepts. */
+export const NOVA2_EMBEDDING_DIMENSIONS = [256, 384, 1024, 3072] as const;
+
+/** Nova 2 synchronous text input limit (characters). */
+export const NOVA2_MAX_TEXT_CHARS = 8192;
+
+/**
+ * Deployed (staging and production) runtimes may only use these values. Anything else fails
+ * environment parsing at startup. Widening this list is a reviewed code change.
+ */
+export const DEPLOYED_SKILL_EMBEDDING_ALLOWLIST = {
+    modelIds: [NOVA2_EMBEDDING_MODEL_ID],
+    dimensions: [1024],
+    regions: ['us-east-1'],
+} as const;
+
+/**
+ * Bump when the embedded text, purposes, truncation or normalisation change. The version is
+ * part of every node key and cache key, so a bump re-embeds skills and invalidates cached
+ * query vectors without touching the stored vector schema.
+ */
+export const SKILL_EMBEDDING_SCHEME_VERSION = 'index-retrieval-v1';
+
+const NOVA2_MODEL_PATTERN = /^amazon\.nova-2-multimodal-embeddings-v(\d{1,3}):(\d{1,3})$/;
+
+export type SkillEmbeddingGeneration = {
+    modelId: string;
+    dimension: number;
+    region: string;
+    /** Stable identity used in node keys and Redis cache keys. */
+    identity: string;
+    /** Neo4j property holding the vector for this generation. */
+    vectorProperty: string;
+    /** Neo4j property holding sha256(identity + text) for idempotent backfill. */
+    keyProperty: string;
+    /** Neo4j vector index over `vectorProperty`. */
+    indexName: string;
+};
+
+export const isNova2EmbeddingModelId = (modelId: string): boolean =>
+    NOVA2_MODEL_PATTERN.test(modelId);
+
+export const isSupportedSkillEmbeddingDimension = (dimension: number): boolean =>
+    (NOVA2_EMBEDDING_DIMENSIONS as readonly number[]).includes(dimension);
+
+export const resolveSkillEmbeddingGeneration = (input: {
+    modelId: string;
+    dimension: number;
+    region: string;
+}): SkillEmbeddingGeneration => {
+    const match = NOVA2_MODEL_PATTERN.exec(input.modelId);
+
+    if (!match) throw new Error('Unsupported skill embedding model');
+    if (!isSupportedSkillEmbeddingDimension(input.dimension)) {
+        throw new Error('Unsupported skill embedding dimension');
+    }
+
+    // Only digits from the validated model ID and dimension reach Cypher identifiers.
+    const slug = `nova2_v${match[1]}_${match[2]}_d${input.dimension}`;
+
+    return {
+        modelId: input.modelId,
+        dimension: input.dimension,
+        region: input.region,
+        identity: `${input.modelId}|${input.dimension}|${SKILL_EMBEDDING_SCHEME_VERSION}`,
+        vectorProperty: `embedding_${slug}`,
+        keyProperty: `embedding_${slug}_key`,
+        indexName: `skill_embedding_${slug}_idx`,
+    };
+};
+
+/** Cypher for the generation's vector index; idempotent so it is safe on every startup. */
+export const buildSkillEmbeddingVectorIndexQuery = (generation: SkillEmbeddingGeneration): string =>
+    `CREATE VECTOR INDEX ${generation.indexName} IF NOT EXISTS FOR (s:Skill) ON (s.${generation.vectorProperty}) OPTIONS {indexConfig: {\`vector.dimensions\`: ${generation.dimension}, \`vector.similarity_function\`: 'cosine'}}`;
+
+/**
+ * Build the Cypher that writes one generation's vector and key (shared with the seed path).
+ *
+ * Compare-and-set on the source: the write lands only while the skill's statement and
+ * description still equal `$sourceStatement` / `$sourceDescription`, the values the vector was
+ * computed from (null is compared as ''). A backfill or live write that embedded text which has
+ * since been edited matches nothing and returns count 0, so it can never replace the newer
+ * text's vector and key. The throwaway property write takes the node's write lock first
+ * (Neo4j's documented lost-update idiom), so the source check and the SET are atomic against a
+ * concurrent skill update.
+ */
+export const buildSkillEmbeddingUpdateQuery = (
+    generation: SkillEmbeddingGeneration,
+    scopedToFramework: boolean
+): string => {
+    const match = scopedToFramework
+        ? 'MATCH (f:SkillFramework {id: $frameworkId})-[:CONTAINS]->(s:Skill {id: $skillId})'
+        : 'MATCH (s:Skill {id: $skillId})';
+
+    // Property names come from resolveSkillEmbeddingGeneration ([a-z0-9_] only).
+    return `${match}
+             SET s._embeddingWriteLock = true
+             REMOVE s._embeddingWriteLock
+             WITH s
+             WHERE coalesce(s.statement, '') = $sourceStatement
+               AND coalesce(s.description, '') = $sourceDescription
+             SET s.${generation.vectorProperty} = $embedding,
+                 s.${generation.keyProperty} = $embeddingKey
+             RETURN count(s) AS count`;
+};
+
+/** The source fields a Nova vector was computed from, as compared by the update query. */
+export type SkillEmbeddingSource = {
+    statement: string;
+    description: string;
+};
+
+export const toSkillEmbeddingSource = (skill: {
+    statement?: string | null;
+    description?: string | null;
+}): SkillEmbeddingSource => ({
+    statement: skill.statement ?? '',
+    description: skill.description ?? '',
+});
+
+/** Query parameters for buildSkillEmbeddingUpdateQuery. */
+export const buildSkillEmbeddingUpdateParams = (update: {
+    skillId: string;
+    frameworkId?: string;
+    embedding: number[];
+    embeddingKey: string;
+    source: SkillEmbeddingSource;
+}) => ({
+    frameworkId: update.frameworkId ?? null,
+    skillId: update.skillId,
+    embedding: update.embedding,
+    embeddingKey: update.embeddingKey,
+    sourceStatement: update.source.statement,
+    sourceDescription: update.source.description,
+});
+
+/** A vector index semantic search can query, with the vector size it accepts. */
+export type SkillEmbeddingSearchIndex = {
+    /** `nova` = the configured Nova 2 generation; `legacy` = the pre-LC-2206 Google index. */
+    source: 'nova' | 'legacy';
+    indexName: string;
+    dimension: number;
+};
+
+/**
+ * Legacy Google gemini-embedding-001 (768-dim) property and index, exactly as on main before
+ * LC-2206. Semantic search keeps reading it (with Google query embeddings) until
+ * SKILL_EMBEDDING_BEDROCK_READ_ENABLED is flipped on, and skill writes keep it current while
+ * a Google key is configured, so flipping the flag back off is the rollback. A follow-up
+ * cleanup drops it after the rollback window.
+ */
+export const LEGACY_GOOGLE_SKILL_EMBEDDING_INDEX = 'skill_embedding_vector_idx';
+export const LEGACY_GOOGLE_SKILL_EMBEDDING_PROPERTY = 'embedding';
+export const LEGACY_GOOGLE_SKILL_EMBEDDING_DIMENSION = 768;
+export const LEGACY_GOOGLE_SKILL_EMBEDDING_MODEL = 'gemini-embedding-001';
+
+export const LEGACY_GOOGLE_SKILL_EMBEDDING_SEARCH_INDEX: SkillEmbeddingSearchIndex = {
+    source: 'legacy',
+    indexName: LEGACY_GOOGLE_SKILL_EMBEDDING_INDEX,
+    dimension: LEGACY_GOOGLE_SKILL_EMBEDDING_DIMENSION,
+};
+
+/** Main's legacy index DDL, unchanged; idempotent so fresh local and e2e databases have it. */
+export const LEGACY_GOOGLE_SKILL_EMBEDDING_VECTOR_INDEX_QUERY = `CREATE VECTOR INDEX ${LEGACY_GOOGLE_SKILL_EMBEDDING_INDEX} IF NOT EXISTS FOR (s:Skill) ON (s.${LEGACY_GOOGLE_SKILL_EMBEDDING_PROPERTY}) OPTIONS {indexConfig: {\`vector.dimensions\`: ${LEGACY_GOOGLE_SKILL_EMBEDDING_DIMENSION}, \`vector.similarity_function\`: 'cosine'}}`;
+
+/** Search index for the configured Nova 2 generation. */
+export const toSkillEmbeddingSearchIndex = (
+    generation: SkillEmbeddingGeneration
+): SkillEmbeddingSearchIndex => ({
+    source: 'nova',
+    indexName: generation.indexName,
+    dimension: generation.dimension,
+});
+
+/**
+ * Staging and production Lambdas do not set NODE_ENV, so a Lambda that is not running under
+ * serverless-offline also counts as deployed.
+ */
+export const isDeployedRuntime = (environment: {
+    NODE_ENV?: string;
+    IS_OFFLINE?: boolean;
+    AWS_LAMBDA_FUNCTION_NAME?: string;
+}): boolean =>
+    !environment.IS_OFFLINE &&
+    (environment.NODE_ENV === 'production' || Boolean(environment.AWS_LAMBDA_FUNCTION_NAME));

package/src/constants/campaigns.ts

@@ -0,0 +1,9 @@
+import { environment } from '@environment';
+
+export const CAMPAIGN_ADMIN_PROFILE_IDS = (environment.CAMPAIGN_ADMIN_PROFILE_IDS ?? '')
+    .split(',')
+    .map(value => value.trim())
+    .filter(Boolean);
+
+export const isCampaignAdmin = (profileId: string): boolean =>
+    CAMPAIGN_ADMIN_PROFILE_IDS.includes(profileId);

package/src/helpers/activity.helpers.ts

@@ -32,9 +32,9 @@ export const logCredentialSent = async (params: {
     listingId?: string;
     source: CredentialActivitySourceType;
     metadata?: Record<string, unknown>;
-    // LC-1644 Phase 1d: callers can pre-generate activityId upfront so
-    // logCredentialSent can run in parallel with dependent writes (e.g. sendBoost
-    // which stores activityId on the SentCredential relationship).
+    // Callers can supply a correlation ID for related graph writes. Finish
+    // activity writes before delivery writes touch the same profile/listing;
+    // sharing an ID does not make parallel auto-commit transactions independent.
     activityId?: string;
 }): Promise<string> => {
     const isInboxSend = params.recipientType === 'email' || params.recipientType === 'phone';

package/src/helpers/app-credential-delivery.helpers.test.ts

@@ -0,0 +1,56 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+const mocks = vi.hoisted(() => ({ activity: vi.fn(), send: vi.fn() }));
+vi.mock('./activity.helpers', () => ({ logCredentialSent: mocks.activity }));
+vi.mock('./boost.helpers', () => ({ sendBoost: mocks.send }));
+
+import { sendAppCredentialWithActivity } from './app-credential-delivery.helpers';
+
+const activity: Parameters<typeof sendAppCredentialWithActivity>[0] = {
+    actorProfileId: 'owner',
+    recipientType: 'profile',
+    recipientIdentifier: 'learner',
+    listingId: 'app',
+    source: 'appEvent',
+};
+const delivery = { listingId: 'app' } as Parameters<typeof sendAppCredentialWithActivity>[1];
+
+describe('app credential delivery graph ordering', () => {
+    beforeEach(() => {
+        vi.resetAllMocks();
+        mocks.send.mockResolvedValue('credential:1');
+    });
+
+    it('waits for the activity write to commit and uses its ID before starting delivery', async () => {
+        let resolve!: (activityId: string) => void;
+        const promise = new Promise<string>(complete => {
+            resolve = complete;
+        });
+        mocks.activity.mockReturnValue(promise);
+        const result = sendAppCredentialWithActivity(activity, delivery);
+        expect(mocks.activity).toHaveBeenCalledExactlyOnceWith(activity);
+        expect(mocks.send).not.toHaveBeenCalled();
+        resolve('activity:1');
+        expect(await result).toBe('credential:1');
+        expect(mocks.send).toHaveBeenCalledExactlyOnceWith({
+            ...delivery,
+            activityId: 'activity:1',
+        });
+    });
+
+    it('does not deliver a credential after an activity write fails', async () => {
+        const error = new Error('activity write failed');
+        mocks.activity.mockRejectedValue(error);
+        await expect(sendAppCredentialWithActivity(activity, delivery)).rejects.toBe(error);
+        expect(mocks.send).not.toHaveBeenCalled();
+    });
+
+    it('propagates delivery failure without retrying activity or delivery', async () => {
+        const error = new Error('delivery failed');
+        mocks.activity.mockResolvedValue('activity:1');
+        mocks.send.mockRejectedValue(error);
+        await expect(sendAppCredentialWithActivity(activity, delivery)).rejects.toBe(error);
+        expect(mocks.activity).toHaveBeenCalledOnce();
+        expect(mocks.send).toHaveBeenCalledOnce();
+    });
+});

package/src/helpers/app-credential-delivery.helpers.ts

@@ -0,0 +1,11 @@
+import { logCredentialSent } from './activity.helpers';
+import { sendBoost } from './boost.helpers';
+
+/** Complete activity graph writes before delivery writes touch the same listing. */
+export const sendAppCredentialWithActivity = async (
+    activity: Parameters<typeof logCredentialSent>[0],
+    delivery: Omit<Parameters<typeof sendBoost>[0], 'activityId'>
+): Promise<string> => {
+    const activityId = await logCredentialSent(activity);
+    return sendBoost({ ...delivery, activityId });
+};

package/src/helpers/bedrock-embedding.helpers.test.ts

@@ -0,0 +1,208 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+const sdk = vi.hoisted(() => ({
+    clientConfigs: [] as unknown[],
+    commands: [] as unknown[],
+    send: vi.fn(),
+}));
+
+vi.mock('@aws-sdk/client-bedrock-runtime', () => ({
+    BedrockRuntimeClient: vi.fn(function BedrockRuntimeClient(config: unknown) {
+        sdk.clientConfigs.push(config);
+        return { send: sdk.send, destroy: vi.fn() };
+    }),
+    InvokeModelCommand: vi.fn(function InvokeModelCommand(input: unknown) {
+        sdk.commands.push(input);
+        return { input };
+    }),
+}));
+
+import {
+    SkillEmbeddingError,
+    buildNovaEmbeddingRequest,
+    createDefaultBedrockInvoke,
+    embedTextWithBedrock,
+    normalizeEmbedding,
+    parseNovaEmbeddingResponse,
+    resetBedrockEmbeddingClients,
+    sanitizeBedrockError,
+    type BedrockEmbeddingConfig,
+    type BedrockInvoke,
+} from './bedrock-embedding.helpers';
+
+const config: BedrockEmbeddingConfig = {
+    modelId: 'amazon.nova-2-multimodal-embeddings-v1:0',
+    region: 'us-east-1',
+    dimension: 4,
+    timeoutMs: 1_000,
+};
+
+const encode = (value: unknown): Uint8Array => new TextEncoder().encode(JSON.stringify(value));
+
+const novaBody = (embedding: unknown, extra: Record<string, unknown> = {}) =>
+    encode({ embeddings: [{ embeddingType: 'TEXT', embedding, ...extra }] });
+
+describe('Nova 2 request shape', () => {
+    it('uses GENERIC_INDEX for stored skills and TEXT_RETRIEVAL for queries', () => {
+        expect(buildNovaEmbeddingRequest('Python', 'index', 1024)).toEqual({
+            schemaVersion: 'nova-multimodal-embed-v1',
+            taskType: 'SINGLE_EMBEDDING',
+            singleEmbeddingParams: {
+                embeddingPurpose: 'GENERIC_INDEX',
+                embeddingDimension: 1024,
+                text: { truncationMode: 'NONE', value: 'Python' },
+            },
+        });
+        expect(
+            buildNovaEmbeddingRequest('python skills', 'query', 1024).singleEmbeddingParams
+                .embeddingPurpose
+        ).toBe('TEXT_RETRIEVAL');
+    });
+});
+
+describe('Nova 2 response parsing', () => {
+    it('returns a unit-normalised vector', () => {
+        const vector = parseNovaEmbeddingResponse(novaBody([3, 0, 4, 0]), 4);
+        expect(vector).toEqual([0.6, 0, 0.8, 0]);
+        expect(Math.hypot(...vector)).toBeCloseTo(1, 10);
+    });
+
+    it.each([
+        ['wrong dimension', novaBody([1, 2, 3])],
+        ['truncated input', novaBody([1, 0, 0, 0], { truncatedCharLength: 10 })],
+        [
+            'non-finite values',
+            encode({ embeddings: [{ embeddingType: 'TEXT', embedding: [1, null, 0, 0] }] }),
+        ],
+        ['zero vector', novaBody([0, 0, 0, 0])],
+        [
+            'non-text embedding',
+            encode({ embeddings: [{ embeddingType: 'IMAGE', embedding: [1, 0, 0, 0] }] }),
+        ],
+        [
+            'multiple embeddings',
+            encode({
+                embeddings: [
+                    { embeddingType: 'TEXT', embedding: [1, 0, 0, 0] },
+                    { embeddingType: 'TEXT', embedding: [0, 1, 0, 0] },
+                ],
+            }),
+        ],
+        ['invalid JSON', new TextEncoder().encode('{not json')],
+    ])('rejects %s', (_name, body) => {
+        expect(() => parseNovaEmbeddingResponse(body, 4)).toThrow(SkillEmbeddingError);
+    });
+
+    it('normalizeEmbedding rejects a zero norm', () => {
+        expect(() => normalizeEmbedding([0, 0])).toThrow(SkillEmbeddingError);
+    });
+});
+
+describe('error sanitisation', () => {
+    const providerError = (name: string, status: number) =>
+        Object.assign(new Error(`secret provider body for skill text "Advanced Parsing"`), {
+            name,
+            $metadata: { httpStatusCode: status },
+        });
+
+    it.each([
+        ['ThrottlingException', 429, 'throttled', true],
+        ['ServiceQuotaExceededException', 429, 'throttled', true],
+        ['AccessDeniedException', 403, 'access_denied', false],
+        ['ServiceUnavailableException', 503, 'unavailable', true],
+        ['ModelTimeoutException', 408, 'unavailable', true],
+        ['ValidationException', 400, 'invalid_input', false],
+        ['InternalServerException', 500, 'failed', false],
+    ])('maps %s to %s without leaking the provider message', (name, status, code, retryable) => {
+        const sanitized = sanitizeBedrockError(providerError(name, status));
+
+        expect(sanitized.code).toBe(code);
+        expect(sanitized.retryable).toBe(retryable);
+        expect(sanitized.message).not.toContain('secret');
+        expect(sanitized.message).not.toContain('Advanced Parsing');
+        expect((sanitized as Error & { cause?: unknown }).cause).toBeUndefined();
+    });
+});
+
+describe('embedTextWithBedrock', () => {
+    beforeEach(() => {
+        sdk.clientConfigs.length = 0;
+        sdk.commands.length = 0;
+        sdk.send.mockReset();
+        resetBedrockEmbeddingClients();
+    });
+
+    it('sends exactly one InvokeModel request with the configured model and purpose', async () => {
+        const invoke = vi.fn<BedrockInvoke>().mockResolvedValue(novaBody([1, 1, 1, 1]));
+
+        const vector = await embedTextWithBedrock('  Data analysis  ', 'query', config, invoke);
+
+        expect(vector).toEqual([0.5, 0.5, 0.5, 0.5]);
+        expect(invoke).toHaveBeenCalledTimes(1);
+        const [request] = invoke.mock.calls[0]!;
+        expect(request.modelId).toBe(config.modelId);
+        expect(JSON.parse(request.body)).toMatchObject({
+            singleEmbeddingParams: {
+                embeddingPurpose: 'TEXT_RETRIEVAL',
+                embeddingDimension: 4,
+                text: { value: 'Data analysis', truncationMode: 'NONE' },
+            },
+        });
+    });
+
+    it('rejects empty and over-long input before calling Bedrock', async () => {
+        const invoke = vi.fn<BedrockInvoke>();
+
+        await expect(embedTextWithBedrock('   ', 'index', config, invoke)).rejects.toMatchObject({
+            code: 'invalid_input',
+        });
+        await expect(
+            embedTextWithBedrock('x'.repeat(8193), 'index', config, invoke)
+        ).rejects.toMatchObject({ code: 'invalid_input' });
+        expect(invoke).not.toHaveBeenCalled();
+    });
+
+    it('times out and aborts the in-flight request', async () => {
+        vi.useFakeTimers();
+        let signal: AbortSignal | undefined;
+        const invoke: BedrockInvoke = (_request, options) => {
+            signal = options.abortSignal;
+            return new Promise(() => undefined);
+        };
+
+        const pending = embedTextWithBedrock('slow', 'index', config, invoke);
+        const assertion = expect(pending).rejects.toMatchObject({ code: 'timeout' });
+        await vi.advanceTimersByTimeAsync(config.timeoutMs + 1);
+        await assertion;
+        expect(signal?.aborted).toBe(true);
+        vi.useRealTimers();
+    });
+
+    it('builds a SigV4-only SDK client with no retries and bounded timeouts', async () => {
+        sdk.send.mockResolvedValue({ body: novaBody([0, 2, 0, 0]) });
+
+        const vector = await embedTextWithBedrock(
+            'Leadership',
+            'index',
+            config,
+            createDefaultBedrockInvoke(config)
+        );
+
+        expect(vector).toEqual([0, 1, 0, 0]);
+        expect(sdk.clientConfigs).toEqual([
+            {
+                region: 'us-east-1',
+                maxAttempts: 1,
+                authSchemePreference: ['sigv4'],
+                requestHandler: { requestTimeout: 1_000, connectionTimeout: 3_000 },
+            },
+        ]);
+        expect(sdk.commands).toEqual([
+            expect.objectContaining({
+                modelId: 'amazon.nova-2-multimodal-embeddings-v1:0',
+                contentType: 'application/json',
+                accept: 'application/json',
+            }),
+        ]);
+    });
+});