@neural-technologies-indonesia/nl2sql 0.0.1

package/README.md

@@ -0,0 +1,110 @@
+# text-to-sql-reasoner
+
+Core reasoning flow for text-to-SQL that can be exposed by any transport (Streamlit, REST API, MCP, etc.).
+
+```
+[Input Port]
+   ↓
+[Prompt Composer]
+   ↓
+[Retrieval (RAG)]
+   ↓
+[Reasoning LLM]
+   ↓
+[Query Candidate]
+   ↓
+[MCP: run_query]
+   ↓
+[Execution Feedback]
+   ├─ slow / failed ─► retry loop
+   └─ success & fast ─► output port
+```
+
+## What lives here (and how it works)
+- Pure orchestrator in TypeScript (Bun-friendly) with clean interfaces for prompt composing, retrieval, LLM reasoning, query execution, and feedback policy.
+- Transport-agnostic: callers plug in their own input/output (UI, REST, MCP, etc.).
+- Defaults: prompt composer (`DefaultPromptComposer`), feedback policy (`SimpleFeedbackPolicy`), MCP executor (`MCPQueryExecutor`), MCP retriever (`MCPMilvusRetriever`), and OpenAI LLM wrapper (`OpenAILLM`).
+- Flow: retrieve context → compose prompt → ask LLM for SQL candidates → execute via MCP `run_query` → accept/retry via feedback policy.
+
+## Example wiring
+```ts
+import {
+  ReasoningPipeline,
+  SimpleFeedbackPolicy,
+  DefaultPromptComposer,
+  OpenAILLM,
+  MCPQueryExecutor,
+} from "@neural-technologies-indonesia/nl2sql";
+
+const pipeline = new ReasoningPipeline({
+  composer: new DefaultPromptComposer(),
+  retriever: /* plug a retriever implementation here */,
+  llm: new OpenAILLM({ model: "gpt-4o-mini" }),
+  executor: new MCPQueryExecutor({
+    runQuery: async (sql) => {
+      // call MCP run_query tool here
+      return { durationMs: 120, rows: 1 };
+    },
+  }),
+  policy: new SimpleFeedbackPolicy({ durationBudgetMs: 5000 }),
+});
+
+const result = await pipeline.run({ question: "How many users signed up today?" });
+console.log(result);
+```
+
+## Port it anywhere
+- REST / gRPC / WebSocket: parse request → build `PromptInput` → call `pipeline.run` → return `FlowOutput`.
+- Streamlit or CLI: same core call; render `logs` for debugging.
+- MCP: wrap `pipeline.run` behind a tool; reuse the included `MCPQueryExecutor` to hit `run_query`.
+
+## Inference backends
+- Implemented: OpenAI (default focus).
+- Future slots: MLX LM (Apple), Ollama, vLLM — add new adapters implementing `ReasoningLLM` without changing the pipeline.
+
+## Env
+- Copy `.env.example` and set `OPENAI_API_KEY` (and optionally `OPENAI_BASE_URL`, `OPENAI_MODEL`).
+
+## Examples
+- `example/bun-server.ts`: starts a Bun HTTP server exposing POST `/query` that runs the pipeline (OpenAI + MCP-backed executor/retriever). Run with `bun run example/bun-server.ts`.
+- `example/streamlit_app.py`: minimal Streamlit UI that POSTs to the Bun server. Configure `API_URL` in `.streamlit/secrets.toml` or edit the default (`http://localhost:8787/query`).
+
+## Use the sibling MCP server (`../mcp-server`) via `mcp-use`
+- That folder ships a FastMCP server exposing `run_query` (Postgres latency) and `milvus_retrieve` (RAG search). Start it with your env: `DATABASE_URL=... OPENAI_API_KEY=... bun run start` from `../mcp-server`.
+- Install the client dependency here: `bun add mcp-use` (or `npm install mcp-use`).
+- If your MCP server lives somewhere else, point the helper there with env vars: `MCP_SERVER_CWD=/path/to/server`, optionally `MCP_SERVER_CMD=bun` and `MCP_SERVER_ARGS="run start"`. The example Bun server picks these up automatically.
+- Wire the pipeline to that server with the provided helper:
+  ```ts
+  import {
+    ReasoningPipeline,
+    DefaultPromptComposer,
+    SimpleFeedbackPolicy,
+    OpenAILLM,
+    createMcpUseAdapters,
+  } from "@neural-technologies-indonesia/nl2sql";
+
+  const { executor, retriever, close } = await createMcpUseAdapters({
+    // optional overrides:
+    // serverCwd: path.resolve("../mcp-server"),
+    // command: "bun",
+    // args: ["run", "start"],
+  });
+
+  const pipeline = new ReasoningPipeline({
+    composer: new DefaultPromptComposer(),
+    retriever,
+    llm: new OpenAILLM({ model: process.env.OPENAI_MODEL ?? "gpt-4o-mini" }),
+    executor,
+    policy: new SimpleFeedbackPolicy({ durationBudgetMs: 5000 }),
+  });
+
+  const result = await pipeline.run({ question: "How many users signed up today?" });
+  console.log(result);
+  await close();
+  ```
+- `createMcpUseAdapters` calls the MCP tools (no duplicated logic here) and returns the MCP-backed executor/retriever pair.
+
+## Next steps
+- Swap in real retriever and LLM clients.
+- Add schema guards to prompt composer (e.g., restrict tables/columns).
+- Extend feedback policy with statistical latency bounds or result validation. 

package/dist/adapters/mcpExecutor.d.ts

@@ -0,0 +1,22 @@
+import type { QueryCandidate, QueryExecutor, ExecutionResult } from "../types.js";
+type MCPClient = {
+    runQuery: (sql: string) => Promise<{
+        durationMs: number;
+        rows?: number;
+    }>;
+    runQueryExplain?: (sql: string) => Promise<{
+        durationMs: number;
+        explain?: string;
+        raw?: string;
+        explainLines?: string[];
+        planningMs?: number;
+        executionMs?: number;
+    }>;
+};
+export declare class MCPQueryExecutor implements QueryExecutor {
+    private readonly client;
+    constructor(client: MCPClient);
+    run(candidate: QueryCandidate): Promise<ExecutionResult>;
+}
+export {};
+//# sourceMappingURL=mcpExecutor.d.ts.map

package/dist/adapters/mcpExecutor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mcpExecutor.d.ts","sourceRoot":"","sources":["../../src/adapters/mcpExecutor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,aAAa,EACb,eAAe,EAChB,MAAM,aAAa,CAAC;AAErB,KAAK,SAAS,GAAG;IACf,QAAQ,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC1E,eAAe,CAAC,EAAE,CAChB,GAAG,EAAE,MAAM,KACR,OAAO,CAAC;QACX,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;QACxB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC,CAAC;CACJ,CAAC;AAGF,qBAAa,gBAAiB,YAAW,aAAa;IACpD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAY;gBAEvB,MAAM,EAAE,SAAS;IAIvB,GAAG,CAAC,SAAS,EAAE,cAAc,GAAG,OAAO,CAAC,eAAe,CAAC;CAuB/D"}

package/dist/adapters/mcpExecutor.js

@@ -0,0 +1,30 @@
+// Thin executor adapter that forwards SQL to an MCP client's run_query tool.
+export class MCPQueryExecutor {
+    client;
+    constructor(client) {
+        this.client = client;
+    }
+    async run(candidate) {
+        try {
+            if (this.client.runQueryExplain) {
+                console.log(candidate.sql, "-> run_query_explain via MCP");
+                const res = await this.client.runQueryExplain(candidate.sql);
+                return {
+                    ok: true,
+                    durationMs: res.durationMs,
+                    explain: res.explain,
+                    rawExplain: res.raw,
+                    explainLines: res.explainLines,
+                    planningMs: res.planningMs,
+                    executionMs: res.executionMs,
+                };
+            }
+            const res = await this.client.runQuery(candidate.sql);
+            return { ok: true, durationMs: res.durationMs, rows: res.rows };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : "Unknown MCP error";
+            return { ok: false, error: message };
+        }
+    }
+}

package/dist/adapters/mcpRetriever.d.ts

@@ -0,0 +1,20 @@
+import type { ContextChunk, PromptInput, Retriever } from "../types.js";
+export type MCPMilvusResult = {
+    id?: string;
+    score?: number;
+    rag_text?: string;
+    metadata?: Record<string, unknown>;
+};
+type MCPMilvusClient = {
+    milvusRetrieve: (query: string, topK?: number) => Promise<MCPMilvusResult[] | string | Record<string, unknown>>;
+};
+export declare class MCPMilvusRetriever implements Retriever {
+    private readonly client;
+    private readonly topK;
+    constructor(client: MCPMilvusClient, options?: {
+        topK?: number;
+    });
+    retrieve(input: PromptInput): Promise<ContextChunk[]>;
+}
+export {};
+//# sourceMappingURL=mcpRetriever.d.ts.map

package/dist/adapters/mcpRetriever.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mcpRetriever.d.ts","sourceRoot":"","sources":["../../src/adapters/mcpRetriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExE,MAAM,MAAM,eAAe,GAAG;IAC5B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,cAAc,EAAE,CACd,KAAK,EAAE,MAAM,EACb,IAAI,CAAC,EAAE,MAAM,KACV,OAAO,CAAC,eAAe,EAAE,GAAG,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACpE,CAAC;AAGF,qBAAa,kBAAmB,YAAW,SAAS;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;IACzC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;gBAElB,MAAM,EAAE,eAAe,EAAE,OAAO,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE;IAK1D,QAAQ,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAY5D"}

package/dist/adapters/mcpRetriever.js

@@ -0,0 +1,116 @@
+// Retriever that calls the MCP milvus_retrieve tool and shapes results into context chunks.
+export class MCPMilvusRetriever {
+    client;
+    topK;
+    constructor(client, options) {
+        this.client = client;
+        this.topK = options?.topK ?? 5;
+    }
+    async retrieve(input) {
+        const query = buildSearchQuery(input);
+        if (!query)
+            return [];
+        try {
+            const results = await this.client.milvusRetrieve(query, this.topK);
+            return normalizeResults(results);
+        }
+        catch {
+            // Retrieval failures should not break the flow; return empty context instead.
+            return [];
+        }
+    }
+}
+// Combine user input fields into a single search string.
+const buildSearchQuery = (input) => {
+    const queryParts = [input.question.trim()];
+    if (input.userContext?.trim()) {
+        queryParts.push(`Context: ${input.userContext.trim()}`);
+    }
+    if (input.constraints?.length) {
+        queryParts.push(`Constraints: ${input.constraints.filter(Boolean).join(" | ")}`);
+    }
+    return queryParts.filter(Boolean).join("\n").trim();
+};
+// Accepts any tool payload shape and converts it into ContextChunks.
+const normalizeResults = (results) => {
+    if (typeof results === "string") {
+        return [{ source: "milvus", snippet: results }];
+    }
+    if (!Array.isArray(results)) {
+        return [{ source: "milvus", snippet: contentToSnippet(results) }];
+    }
+    return results.map((item, index) => {
+        const snippetParts = [item.rag_text ?? "(no rag_text)"];
+        if (item.metadata) {
+            snippetParts.push(buildMetadataSnippet(item.metadata));
+        }
+        return {
+            source: item.id ? String(item.id) : `milvus-${index + 1}`,
+            snippet: snippetParts.join("\n"),
+            score: item.score,
+        };
+    });
+};
+const contentToSnippet = (content) => {
+    if (typeof content === "string")
+        return content;
+    if (Array.isArray(content))
+        return content.map((item) => JSON.stringify(item)).join("\n");
+    if (content && typeof content === "object")
+        return JSON.stringify(content);
+    return String(content ?? "");
+};
+const normalizeTableReference = (schema, table) => {
+    if (!table.includes(".")) {
+        return { schema, table };
+    }
+    const parts = table.split(".").filter(Boolean);
+    if (!parts.length) {
+        return { schema, table };
+    }
+    if (parts.length >= 2 && !schema) {
+        return {
+            schema: parts[parts.length - 2],
+            table: parts[parts.length - 1],
+        };
+    }
+    return { schema, table: parts[parts.length - 1] };
+};
+// Formats metadata for the prompt while avoiding fully qualified names that can confuse the model.
+const buildMetadataSnippet = (metadata) => {
+    const parts = [];
+    const schema = typeof metadata.schema === "string" ? metadata.schema : "";
+    const tableRaw = metadata.table ?? metadata.table_name ?? metadata.name;
+    const table = typeof tableRaw === "string" ? tableRaw : "";
+    const normalized = normalizeTableReference(schema, table);
+    const qualified = [normalized.schema, normalized.table].filter(Boolean).join(".");
+    if (qualified)
+        parts.push(`table=${qualified}`);
+    const tableType = typeof metadata.table_type === "string"
+        ? metadata.table_type
+        : typeof metadata.tableType === "string"
+            ? metadata.tableType
+            : "";
+    if (tableType)
+        parts.push(`type=${tableType}`);
+    const primaryKeys = Array.isArray(metadata.primary_keys) ? metadata.primary_keys : [];
+    if (primaryKeys.length)
+        parts.push(`primary_keys=${primaryKeys.map(String).join(",")}`);
+    const columns = Array.isArray(metadata.columns)
+        ? metadata.columns
+            .map((col) => (col && typeof col === "object" && "name" in col ? String(col.name) : null))
+            .filter((name) => Boolean(name))
+        : [];
+    if (columns.length)
+        parts.push(`columns=${columns.join(", ")}`);
+    const description = typeof metadata.description === "string" ? metadata.description : "";
+    if (description)
+        parts.push(`desc=${description}`);
+    if (!parts.length) {
+        const sanitized = { ...metadata };
+        delete sanitized.fully_qualified_name;
+        delete sanitized.fullyQualifiedName;
+        return `metadata=${JSON.stringify(sanitized)}`;
+    }
+    return parts.join("; ");
+};

package/dist/adapters/openaiLLM.d.ts

@@ -0,0 +1,17 @@
+import OpenAI from "openai";
+import type { PromptPayload, QueryCandidate, ReasoningLLM } from "../types.js";
+export type OpenAIOptions = {
+    client?: OpenAI;
+    apiKey?: string;
+    baseURL?: string;
+    model?: string;
+    temperature?: number;
+};
+export declare class OpenAILLM implements ReasoningLLM {
+    private readonly client;
+    private readonly model;
+    private readonly temperature;
+    constructor(options?: OpenAIOptions);
+    generate(payload: PromptPayload): Promise<QueryCandidate[]>;
+}
+//# sourceMappingURL=openaiLLM.d.ts.map

package/dist/adapters/openaiLLM.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"openaiLLM.d.ts","sourceRoot":"","sources":["../../src/adapters/openaiLLM.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE/E,MAAM,MAAM,aAAa,GAAG;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAGF,qBAAa,SAAU,YAAW,YAAY;IAC5C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;gBAEzB,OAAO,GAAE,aAAkB;IAWjC,QAAQ,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;CAwBlE"}

package/dist/adapters/openaiLLM.js

@@ -0,0 +1,57 @@
+import OpenAI from "openai";
+// OpenAI-backed LLM that returns a single SQL candidate from a prompt.
+export class OpenAILLM {
+    client;
+    model;
+    temperature;
+    constructor(options = {}) {
+        this.client =
+            options.client ??
+                new OpenAI({
+                    apiKey: options.apiKey ?? process.env.OPENAI_API_KEY,
+                    baseURL: options.baseURL ?? process.env.OPENAI_BASE_URL,
+                });
+        this.model = options.model ?? process.env.OPENAI_MODEL ?? "gpt-4o-mini";
+        this.temperature = options.temperature ?? 0;
+    }
+    async generate(payload) {
+        const completion = await this.client.responses.create({
+            model: this.model,
+            input: [
+                {
+                    role: "user",
+                    content: [{ type: "input_text", text: payload.prompt }],
+                },
+            ],
+            temperature: this.temperature,
+        });
+        const text = extractText(completion);
+        const sql = text?.trim();
+        return sql
+            ? [
+                {
+                    sql,
+                    rationale: "OpenAI single-shot",
+                },
+            ]
+            : [];
+    }
+}
+// Pulls output text from the OpenAI Responses payload regardless of shape.
+const extractText = (completion) => {
+    if (!completion || typeof completion !== "object")
+        return "";
+    const typed = completion;
+    if (typeof typed.output_text === "string")
+        return typed.output_text;
+    if (Array.isArray(typed.output)) {
+        for (const item of typed.output) {
+            if (item && item.type === "message" && Array.isArray(item.content)) {
+                const textPart = item.content.find((c) => c?.type === "output_text");
+                if (textPart && textPart.type === "output_text")
+                    return textPart.text;
+            }
+        }
+    }
+    return "";
+};

package/dist/clients/mcpUseClient.d.ts

@@ -0,0 +1,39 @@
+import { MCPClient } from "mcp-use";
+import { MCPQueryExecutor } from "../adapters/mcpExecutor.js";
+import { MCPMilvusRetriever } from "../adapters/mcpRetriever.js";
+import type { MCPMilvusResult } from "../adapters/mcpRetriever.js";
+export type MCPUseClientOptions = {
+    serverName?: string;
+    serverCwd?: string;
+    command?: string;
+    args?: string[];
+    env?: Record<string, string>;
+};
+export declare const createMcpUseAdapters: (options?: MCPUseClientOptions) => Promise<{
+    executor: MCPQueryExecutor;
+    retriever: MCPMilvusRetriever;
+    callTool: (name: "run_query" | "run_query_explain" | "list_indexes" | "milvus_retrieve", args: Record<string, unknown>) => Promise<string | MCPMilvusResult[] | Record<string, unknown>>;
+    runQueryExplain: (sql: string) => Promise<{
+        durationMs: number;
+        explainLines: string[] | undefined;
+        planningMs: number | undefined;
+        executionMs: number | undefined;
+        raw: string;
+        explain?: undefined;
+    } | {
+        durationMs: number;
+        explain: string;
+        raw: string;
+        explainLines?: undefined;
+        planningMs?: undefined;
+        executionMs?: undefined;
+    }>;
+    listIndexes: (options?: {
+        schemaname?: string;
+        tablename?: string;
+    }) => Promise<string>;
+    close: () => Promise<void>;
+    client: MCPClient;
+    session: import("mcp-use").MCPSession;
+}>;
+//# sourceMappingURL=mcpUseClient.d.ts.map

package/dist/clients/mcpUseClient.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mcpUseClient.d.ts","sourceRoot":"","sources":["../../src/clients/mcpUseClient.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAC;AACjE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAGnE,MAAM,MAAM,mBAAmB,GAAG;IAChC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B,CAAC;AAwEF,eAAO,MAAM,oBAAoB,GAAU,UAAU,mBAAmB;;;qBA2C9D,WAAW,GAAG,mBAAmB,GAAG,cAAc,GAAG,iBAAiB,QACtE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC5B,OAAO,CAAC,MAAM,GAAG,eAAe,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;2BAuB5B,MAAM;;;;;;;;;;;;;;;4BAoDL;QACnC,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB;;;;EA2BF,CAAC"}

package/dist/clients/mcpUseClient.js

@@ -0,0 +1,182 @@
+import fs from "node:fs";
+import path from "node:path";
+import { MCPClient } from "mcp-use";
+import { MCPQueryExecutor } from "../adapters/mcpExecutor.js";
+import { MCPMilvusRetriever } from "../adapters/mcpRetriever.js";
+// Accepts a whitespace-delimited env var and turns it into an args array.
+const parseArgs = (value) => {
+    if (!value)
+        return undefined;
+    return value
+        .split(" ")
+        .map((part) => part.trim())
+        .filter(Boolean);
+};
+// Convert relative file-like args to absolute paths so Bun can locate the entrypoint
+// even if the MCP client spawns from a different working directory.
+const absolutizeArgs = (args, baseDir) => {
+    return args.map((arg) => {
+        if (path.isAbsolute(arg))
+            return arg;
+        const candidate = path.join(baseDir, arg);
+        return fs.existsSync(candidate) ? candidate : arg;
+    });
+};
+// Extracts the textual part from any MCP tool result content.
+const contentToText = (content) => {
+    if (typeof content === "string")
+        return content;
+    if (!content)
+        return "";
+    if (Array.isArray(content)) {
+        return content
+            .map((item) => {
+            if (typeof item === "string")
+                return item;
+            if (item && typeof item === "object" && "text" in item) {
+                return String(item.text);
+            }
+            return JSON.stringify(item);
+        })
+            .join("\n");
+    }
+    if (typeof content === "object" &&
+        "text" in content) {
+        return String(content.text);
+    }
+    return JSON.stringify(content);
+};
+const toolResultPayload = (result) => {
+    if (!result)
+        return "";
+    if (typeof result === "object" && result !== null) {
+        const typed = result;
+        if (typed.isError) {
+            const message = contentToText(typed.content ?? result) || "MCP tool returned an error";
+            throw new Error(message);
+        }
+        if (typed.structuredContent)
+            return typed.structuredContent;
+        if (typed.content)
+            return contentToText(typed.content);
+    }
+    return contentToText(result);
+};
+const parseDurationMs = (text) => {
+    const match = text.match(/([0-9]+(?:\.[0-9]+)?)\s*ms/i);
+    return match ? Number(match[1]) : undefined;
+};
+// Starts mcp-use against the sibling MCP server and returns MCP-backed adapters.
+export const createMcpUseAdapters = async (options) => {
+    const serverName = options?.serverName ?? "query-runner";
+    console.log("mcp server cwd", process.env.MCP_SERVER_CWD);
+    const serverCwd = options?.serverCwd ??
+        process.env.MCP_SERVER_CWD ??
+        path.resolve(process.cwd(), "..", "mcp-server");
+    const command = options?.command ?? process.env.MCP_SERVER_CMD ?? "bun";
+    const rawArgs = options?.args ??
+        parseArgs(process.env.MCP_SERVER_ARGS) ?? ["run", "start"];
+    const args = absolutizeArgs(rawArgs, serverCwd);
+    if (!fs.existsSync(serverCwd)) {
+        throw new Error(`MCP server directory not found at ${serverCwd}. Set MCP_SERVER_CWD or pass options.serverCwd to point at your MCP server (see README).`);
+    }
+    const client = new MCPClient({
+        mcpServers: {
+            [serverName]: {
+                command,
+                args,
+                cwd: serverCwd,
+                env: options?.env,
+            },
+        },
+    });
+    console.log(`Starting MCP server "${serverName}" via ${command} ${args.join(" ")} (cwd: ${serverCwd})`);
+    await client.createAllSessions();
+    const session = client.getSession(serverName);
+    if (!session) {
+        throw new Error(`Failed to start MCP session for ${serverName}`);
+    }
+    const callTool = async (name, args) => {
+        const result = await session.callTool(name, args);
+        return toolResultPayload(result);
+    };
+    const executor = new MCPQueryExecutor({
+        runQuery: async (sql) => {
+            const content = await callTool("run_query", { raw_query: sql });
+            const text = contentToText(content);
+            const durationMs = parseDurationMs(text) ?? Number.NaN; // Keep numeric for typing even if parse fails.
+            return { durationMs };
+        },
+        runQueryExplain: async (sql) => runQueryExplain(sql),
+    });
+    const retriever = new MCPMilvusRetriever({
+        milvusRetrieve: async (query, topK) => callTool("milvus_retrieve", { query, top_k: topK ?? 5 }),
+    });
+    const runQueryExplain = async (sql) => {
+        const content = await callTool("run_query_explain", { raw_query: sql });
+        const text = contentToText(content);
+        const durationMs = parseDurationMs(text) ?? Number.NaN;
+        const jsonStart = text.search(/JSON:\s*/i);
+        const explainMarker = text.search(/EXPLAIN ANALYZE:\s*/i);
+        let parsed = null;
+        if (jsonStart !== -1) {
+            const jsonBlock = text
+                .slice(jsonStart)
+                .replace(/^JSON:\s*/i, "");
+            const jsonText = explainMarker !== -1
+                ? jsonBlock.slice(0, explainMarker - jsonStart).trim()
+                : jsonBlock.trim();
+            try {
+                parsed = JSON.parse(jsonText);
+            }
+            catch {
+                parsed = null;
+            }
+        }
+        if (parsed) {
+            const explainLines = Array.isArray(parsed.explainLines)
+                ? parsed.explainLines.map((line) => String(line))
+                : undefined;
+            return {
+                durationMs: typeof parsed.durationMs === "number"
+                    ? parsed.durationMs
+                    : durationMs,
+                explainLines,
+                planningMs: typeof parsed.planningMs === "number"
+                    ? parsed.planningMs
+                    : undefined,
+                executionMs: typeof parsed.executionMs === "number"
+                    ? parsed.executionMs
+                    : undefined,
+                raw: text,
+            };
+        }
+        const explainSplit = text.split(/EXPLAIN ANALYZE:\s*/i)[1] ??
+            text.split(/EXPLAIN:\s*/i)[1] ??
+            "";
+        return { durationMs, explain: explainSplit.trim(), raw: text };
+    };
+    const listIndexes = async (options) => {
+        const content = await callTool("list_indexes", {
+            schemaname: options?.schemaname,
+            tablename: options?.tablename,
+        });
+        return contentToText(content);
+    };
+    const close = async () => {
+        if (typeof client
+            .closeAllSessions === "function") {
+            await client.closeAllSessions();
+        }
+    };
+    return {
+        executor,
+        retriever,
+        callTool,
+        runQueryExplain,
+        listIndexes,
+        close,
+        client,
+        session,
+    };
+};

package/dist/defaults/promptComposer.d.ts

@@ -0,0 +1,7 @@
+import type { ContextChunk, PromptComposer, PromptInput, PromptPayload } from "../types.js";
+export declare class DefaultPromptComposer implements PromptComposer {
+    private readonly systemPrompt;
+    constructor(systemPrompt?: string);
+    compose(input: PromptInput, retrieved: ContextChunk[]): Promise<PromptPayload>;
+}
+//# sourceMappingURL=promptComposer.d.ts.map

package/dist/defaults/promptComposer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"promptComposer.d.ts","sourceRoot":"","sources":["../../src/defaults/promptComposer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAW5F,qBAAa,qBAAsB,YAAW,cAAc;IAC1D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,YAAY,SAAqB;IAIvC,OAAO,CAAC,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,aAAa,CAAC;CAsBrF"}

package/dist/defaults/promptComposer.js

@@ -0,0 +1,33 @@
+const defaultInstruction = [
+    "You are a text-to-SQL planner.",
+    "Use the provided context to stay within available schema; only use table and column names that appear in the context.",
+    "Return only SQL and keep it safe for read-only execution.",
+    "Output a single SELECT/CTE statement with no trailing semicolon, no markdown/code fences, and no extra commentary.",
+    "If the context is empty or does not mention any relevant tables/columns for the question, do not guess; instead return: SELECT 'NO_RELEVANT_TABLE'::text AS reason",
+].join(" ");
+// Builds a simple text prompt that threads user input and retrieved context.
+export class DefaultPromptComposer {
+    systemPrompt;
+    constructor(systemPrompt = defaultInstruction) {
+        this.systemPrompt = systemPrompt;
+    }
+    async compose(input, retrieved) {
+        const contextBlock = retrieved
+            .map((c, idx) => `#${idx + 1} [${c.source}] (score=${c.score ?? "n/a"})\n${c.snippet}`)
+            .join("\n\n");
+        const constraints = input.constraints?.length ? `Constraints:\n- ${input.constraints.join("\n- ")}` : "";
+        const prompt = [
+            this.systemPrompt,
+            constraints,
+            "User question:",
+            input.question,
+            input.userContext ? `User context: ${input.userContext}` : "",
+            "Context:",
+            contextBlock || "(none)",
+            "Return a best-effort SQL statement.",
+        ]
+            .filter(Boolean)
+            .join("\n\n");
+        return { prompt, context: retrieved };
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,8 @@
+export * from "./types.js";
+export * from "./pipeline.js";
+export * from "./defaults/promptComposer.js";
+export * from "./adapters/mcpExecutor.js";
+export * from "./adapters/mcpRetriever.js";
+export * from "./adapters/openaiLLM.js";
+export * from "./clients/mcpUseClient.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,8BAA8B,CAAC;AAC7C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,4BAA4B,CAAC;AAC3C,cAAc,yBAAyB,CAAC;AACxC,cAAc,2BAA2B,CAAC"}

package/dist/index.js

@@ -0,0 +1,7 @@
+export * from "./types.js";
+export * from "./pipeline.js";
+export * from "./defaults/promptComposer.js";
+export * from "./adapters/mcpExecutor.js";
+export * from "./adapters/mcpRetriever.js";
+export * from "./adapters/openaiLLM.js";
+export * from "./clients/mcpUseClient.js";

package/dist/pipeline.d.ts

@@ -0,0 +1,29 @@
+import type { ExecutionResult, FeedbackPolicy, FlowConfig, FlowOutput, IndexLookup, PromptComposer, PromptInput, QueryExecutor, ReasoningLLM, Retriever } from "./types.js";
+export declare class ReasoningPipeline {
+    private readonly composer;
+    private readonly retriever;
+    private readonly llm;
+    private readonly executor;
+    private readonly policy;
+    private readonly indexLookup?;
+    private readonly config;
+    constructor(options: {
+        composer: PromptComposer;
+        retriever: Retriever;
+        llm: ReasoningLLM;
+        executor: QueryExecutor;
+        policy: FeedbackPolicy;
+        indexLookup?: IndexLookup;
+        config?: FlowConfig;
+    });
+    run(input: PromptInput): Promise<FlowOutput>;
+}
+export declare class SimpleFeedbackPolicy implements FeedbackPolicy {
+    private readonly durationBudgetMs;
+    constructor(options?: {
+        durationBudgetMs?: number;
+    });
+    accept(result: ExecutionResult): boolean;
+    shouldRetry(result: ExecutionResult, attempt: number): boolean;
+}
+//# sourceMappingURL=pipeline.d.ts.map

package/dist/pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,eAAe,EACf,cAAc,EACd,UAAU,EAEV,UAAU,EACV,WAAW,EACX,cAAc,EACd,WAAW,EAEX,aAAa,EACb,YAAY,EACZ,SAAS,EACV,MAAM,YAAY,CAAC;AAQpB,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAiB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IACtC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAgB;IACzC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAc;IAC3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuB;gBAElC,OAAO,EAAE;QACnB,QAAQ,EAAE,cAAc,CAAC;QACzB,SAAS,EAAE,SAAS,CAAC;QACrB,GAAG,EAAE,YAAY,CAAC;QAClB,QAAQ,EAAE,aAAa,CAAC;QACxB,MAAM,EAAE,cAAc,CAAC;QACvB,WAAW,CAAC,EAAE,WAAW,CAAC;QAC1B,MAAM,CAAC,EAAE,UAAU,CAAC;KACrB;IAWK,GAAG,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC;CAkDnD;AAGD,qBAAa,oBAAqB,YAAW,cAAc;IACzD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,OAAO,CAAC,EAAE;QAAE,gBAAgB,CAAC,EAAE,MAAM,CAAA;KAAE;IAInD,MAAM,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO;IAIxC,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO;CAK/D"}

package/dist/pipeline.js

@@ -0,0 +1,84 @@
+// Default config values used when a field is not provided.
+const defaultConfig = {
+    maxAttempts: 3,
+};
+// Orchestrates the reasoning flow: retrieve → compose → LLM → execute with feedback-driven retries.
+export class ReasoningPipeline {
+    composer;
+    retriever;
+    llm;
+    executor;
+    policy;
+    indexLookup;
+    config;
+    constructor(options) {
+        this.composer = options.composer;
+        this.retriever = options.retriever;
+        this.llm = options.llm;
+        this.executor = options.executor;
+        this.policy = options.policy;
+        this.indexLookup = options.indexLookup;
+        this.config = { ...defaultConfig, ...options.config };
+    }
+    // Runs a single question through the pipeline and returns the outcome plus attempt logs.
+    async run(input) {
+        const baseConstraints = [...(input.constraints ?? [])];
+        if (this.indexLookup && input.indexFilters) {
+            const indexInfo = await this.indexLookup.listIndexes(input.indexFilters);
+            if (indexInfo.trim()) {
+                baseConstraints.push(`Available indexes:\n${indexInfo}`);
+            }
+        }
+        const logs = [];
+        const attemptsLimit = this.config.maxAttempts;
+        let lastError;
+        let lastCandidate;
+        for (let attempt = 0; attempt < attemptsLimit; attempt++) {
+            const attemptNumber = attempt + 1;
+            const constraints = [...baseConstraints];
+            if (lastError) {
+                constraints.push(`Previous attempt error:\n${lastError}`);
+            }
+            if (lastCandidate?.sql) {
+                constraints.push(`Previous attempt SQL:\n${lastCandidate.sql}`);
+            }
+            const attemptInput = { ...input, constraints };
+            const context = await this.retriever.retrieve(attemptInput);
+            const prompt = await this.composer.compose(attemptInput, context);
+            const candidates = await this.llm.generate(prompt);
+            const candidate = candidates[0];
+            if (!candidate)
+                break;
+            const execution = await this.executor.run(candidate);
+            logs.push({ attempt: attemptNumber, candidate, execution });
+            if (this.policy.accept(execution)) {
+                return { accepted: true, candidate, result: execution, logs };
+            }
+            const shouldRetry = this.policy.shouldRetry(execution, attemptNumber);
+            const hasAttemptsLeft = attemptNumber < attemptsLimit;
+            if (!shouldRetry || !hasAttemptsLeft) {
+                return { accepted: false, candidate, result: execution, logs };
+            }
+            lastError = execution.error;
+            lastCandidate = candidate;
+        }
+        return { accepted: false, logs };
+    }
+}
+// Simple policy: succeed if ok and under a latency budget; otherwise allow a few retries.
+export class SimpleFeedbackPolicy {
+    durationBudgetMs;
+    constructor(options) {
+        this.durationBudgetMs = options?.durationBudgetMs ?? 5000;
+    }
+    accept(result) {
+        return result.ok && (result.durationMs ?? Infinity) <= this.durationBudgetMs;
+    }
+    shouldRetry(result, attempt) {
+        if (result.ok)
+            return false;
+        if ((result.durationMs ?? Infinity) > this.durationBudgetMs)
+            return attempt < 2;
+        return attempt < 3;
+    }
+}

package/dist/types.d.ts

@@ -0,0 +1,68 @@
+export type ContextChunk = {
+    source: string;
+    snippet: string;
+    score?: number;
+};
+export type PromptInput = {
+    question: string;
+    userContext?: string;
+    constraints?: string[];
+    indexFilters?: IndexFilters;
+};
+export type PromptPayload = {
+    prompt: string;
+    context: ContextChunk[];
+};
+export type QueryCandidate = {
+    sql: string;
+    rationale?: string;
+};
+export type ExecutionResult = {
+    ok: boolean;
+    durationMs?: number;
+    rows?: number;
+    error?: string;
+    explain?: string;
+    rawExplain?: string;
+    explainLines?: string[];
+    planningMs?: number;
+    executionMs?: number;
+};
+export interface PromptComposer {
+    compose(input: PromptInput, retrieved: ContextChunk[]): Promise<PromptPayload>;
+}
+export interface Retriever {
+    retrieve(input: PromptInput): Promise<ContextChunk[]>;
+}
+export interface ReasoningLLM {
+    generate(prompt: PromptPayload): Promise<QueryCandidate[]>;
+}
+export interface QueryExecutor {
+    run(candidate: QueryCandidate): Promise<ExecutionResult>;
+}
+export type IndexFilters = {
+    schemaname?: string;
+    tablename?: string;
+};
+export interface IndexLookup {
+    listIndexes(filters?: IndexFilters): Promise<string>;
+}
+export interface FeedbackPolicy {
+    accept(result: ExecutionResult): boolean;
+    shouldRetry(result: ExecutionResult, attempt: number): boolean;
+}
+export type FlowConfig = {
+    maxAttempts?: number;
+};
+export type FlowLog = {
+    attempt: number;
+    candidate: QueryCandidate;
+    execution: ExecutionResult;
+};
+export type FlowOutput = {
+    accepted: boolean;
+    candidate?: QueryCandidate;
+    result?: ExecutionResult;
+    logs: FlowLog[];
+};
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AACA,MAAM,MAAM,YAAY,GAAG;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,YAAY,EAAE,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAGF,MAAM,MAAM,eAAe,GAAG;IAC5B,EAAE,EAAE,OAAO,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAGF,MAAM,WAAW,cAAc;IAC7B,OAAO,CACL,KAAK,EAAE,WAAW,EAClB,SAAS,EAAE,YAAY,EAAE,GACxB,OAAO,CAAC,aAAa,CAAC,CAAC;CAC3B;AAED,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;CACvD;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;CAC5D;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,CAAC,SAAS,EAAE,cAAc,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;CAC1D;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,WAAW,WAAW;IAC1B,WAAW,CAAC,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACtD;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC;IACzC,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC;CAChE;AAED,MAAM,MAAM,UAAU,GAAG;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAGF,MAAM,MAAM,OAAO,GAAG;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,cAAc,CAAC;IAC1B,SAAS,EAAE,eAAe,CAAC;CAC5B,CAAC;AAGF,MAAM,MAAM,UAAU,GAAG;IACvB,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,cAAc,CAAC;IAC3B,MAAM,CAAC,EAAE,eAAe,CAAC;IACzB,IAAI,EAAE,OAAO,EAAE,CAAC;CACjB,CAAC"}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@neural-technologies-indonesia/nl2sql",
+  "version": "0.0.1",
+  "description": "Core reasoning pipeline for text-to-SQL.",
+  "private": false,
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "sideEffects": false,
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "lint": "echo \"add linter\"",
+    "test": "echo \"no tests yet\""
+  },
+  "dependencies": {
+    "langchain": "^1.2.1",
+    "mcp-use": "*",
+    "openai": "^6.14.0"
+  },
+  "devDependencies": {
+    "@types/bun": "latest"
+  },
+  "peerDependencies": {
+    "typescript": "^5"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}