@autorag/librarian 2.4.2 → 2.5.0

package/licenses/gemma-terms-of-use.txt

@@ -0,0 +1,158 @@
+Gemma Terms of Use
+
+Source: https://ai.google.dev/gemma/terms
+Retrieved for this compliance bundle on 2026-09-14. The text below is copied from the current source page.
+
+The terms below apply to Gemma models listed in the Appendix at bottom of this page. For Gemma 4 terms, see the Gemma 4 license.
+
+Last modified: April 1, 2026
+
+By using, reproducing, modifying, distributing, performing or displaying any portion or element of Gemma, Model Derivatives including via any Hosted Service, (each as defined below) (collectively, the "Gemma Services") or otherwise accepting the terms of this Agreement, you agree to be bound by this Agreement.
+
+Section 1: DEFINITIONS
+
+1.1 Definitions
+
+(a) "Agreement" or "Gemma Terms of Use" means these terms and conditions that govern the use, reproduction, Distribution or modification of the Gemma Services and any terms and conditions incorporated by reference.
+
+(b) "Distribution" or "Distribute" means any transmission, publication, or other sharing of Gemma or Model Derivatives to a third party, including by providing or making Gemma or its functionality available as a hosted service via API, web access, or any other electronic or remote means ("Hosted Service").
+
+(c) "Gemma" means the set of machine learning language models, trained model weights and parameters identified in the Appendix, regardless of the source that you obtained it from.
+
+(d) "Google" means Google LLC.
+
+(e) "Model Derivatives" means all (i) modifications to Gemma, (ii) works based on Gemma, or (iii) any other machine learning model which is created by transfer of patterns of the weights, parameters, operations, or Output of Gemma, to that model in order to cause that model to perform similarly to Gemma, including distillation methods that use intermediate data representations or methods based on the generation of synthetic data Outputs by Gemma for training that model. For clarity, Outputs are not deemed Model Derivatives.
+
+(f) "Output" means the information content output of Gemma or a Model Derivative that results from operating or otherwise using Gemma or the Model Derivative, including via a Hosted Service.
+
+1.2
+
+As used in this Agreement, "including" means "including without limitation".
+
+Section 2: ELIGIBILITY AND USAGE
+
+2.1 Eligibility
+
+You represent and warrant that you have the legal capacity to enter into this Agreement (including being of sufficient age of consent). If you are accessing or using any of the Gemma Services for or on behalf of a legal entity, (a) you are entering into this Agreement on behalf of yourself and that legal entity, (b) you represent and warrant that you have the authority to act on behalf of and bind that entity to this Agreement and (c) references to "you" or "your" in the remainder of this Agreement refers to both you (as an individual) and that entity.
+
+2.2 Use
+
+You may use, reproduce, modify, Distribute, perform or display any of the Gemma Services only in accordance with the terms of this Agreement, and must not violate (or encourage or permit anyone else to violate) any term of this Agreement.
+
+Section 3: DISTRIBUTION AND RESTRICTIONS
+
+3.1 Distribution and Redistribution
+
+You may reproduce or Distribute copies of Gemma or Model Derivatives if you meet all of the following conditions:
+
+You must include the use restrictions referenced in Section 3.2 as an enforceable provision in any agreement (e.g., license agreement, terms of use, etc.) governing the use and/or distribution of Gemma or Model Derivatives and you must provide notice to subsequent users you Distribute to that Gemma or Model Derivatives are subject to the use restrictions in Section 3.2.
+
+You must provide all third party recipients of Gemma or Model Derivatives a copy of this Agreement.
+
+You must cause any modified files to carry prominent notices stating that you modified the files.
+
+All Distributions (other than through a Hosted Service) must be accompanied by a "Notice" text file that contains the following notice: "Gemma is provided under and subject to the Gemma Terms of Use found at ai.google.dev/gemma/terms".
+
+You may add your own intellectual property statement to your modifications and, except as set forth in this Section, may provide additional or different terms and conditions for use, reproduction, or Distribution of your modifications, or for any such Model Derivatives as a whole, provided your use, reproduction, modification, Distribution, performance, and display of Gemma otherwise complies with the terms and conditions of this Agreement. Any additional or different terms and conditions you impose must not conflict with the terms of this Agreement.
+
+3.2 Use Restrictions
+
+You must not use any of the Gemma Services:
+
+for the restricted uses set forth in the Gemma Prohibited Use Policy at ai.google.dev/gemma/prohibited_use_policy ("Prohibited Use Policy"), which is hereby incorporated by reference into this Agreement; or
+
+in violation of applicable laws and regulations.
+
+To the maximum extent permitted by law, Google reserves the right to restrict (remotely or otherwise) usage of any of the Gemma Services that Google reasonably believes are in violation of this Agreement.
+
+3.3 Generated Output
+
+Google claims no rights in Outputs you generate using Gemma. You and your users are solely responsible for Outputs and their subsequent uses.
+
+Section 4: ADDITIONAL PROVISIONS
+
+4.1 Updates
+
+Google may update Gemma from time to time.
+
+4.2 Trademarks
+
+Nothing in this Agreement grants you any rights to use Google's trademarks, trade names, logos or to otherwise suggest endorsement or misrepresent the relationship between you and Google. Google reserves any rights not expressly granted herein.
+
+4.3 DISCLAIMER OF WARRANTY
+
+UNLESS REQUIRED BY APPLICABLE LAW, THE GEMMA SERVICES, AND OUTPUTS, ARE PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. YOU ARE SOLELY RESPONSIBLE FOR DETERMINING THE APPROPRIATENESS OF USING, REPRODUCING, MODIFYING, PERFORMING, DISPLAYING OR DISTRIBUTING ANY OF THE GEMMA SERVICES OR OUTPUTS AND ASSUME ANY AND ALL RISKS ASSOCIATED WITH YOUR USE OR DISTRIBUTION OF ANY OF THE GEMMA SERVICES OR OUTPUTS AND YOUR EXERCISE OF RIGHTS AND PERMISSIONS UNDER THIS AGREEMENT.
+
+4.4 LIMITATION OF LIABILITY
+
+TO THE FULLEST EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO LEGAL THEORY, WHETHER IN TORT (INCLUDING NEGLIGENCE), PRODUCT LIABILITY, CONTRACT, OR OTHERWISE, UNLESS REQUIRED BY APPLICABLE LAW, SHALL GOOGLE OR ITS AFFILIATES BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, EXEMPLARY, CONSEQUENTIAL, OR PUNITIVE DAMAGES, OR LOST PROFITS OF ANY KIND ARISING FROM THIS AGREEMENT OR RELATED TO, ANY OF THE GEMMA SERVICES OR OUTPUTS EVEN IF GOOGLE OR ITS AFFILIATES HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+4.5 Term, Termination, and Survival
+
+The term of this Agreement will commence upon your acceptance of this Agreement (including acceptance by your use, modification, or Distribution, reproduction, performance or display of any portion or element of the Gemma Services) and will continue in full force and effect until terminated in accordance with the terms of this Agreement. Google may terminate this Agreement if you are in breach of any term of this Agreement. Upon termination of this Agreement, you must delete and cease use and Distribution of all copies of Gemma and Model Derivatives in your possession or control. Sections 1, 2.1, 3.3, 4.2 to 4.9 shall survive the termination of this Agreement.
+
+4.6 Governing Law and Jurisdiction
+
+This Agreement will be governed by the laws of the State of California without regard to choice of law principles. The UN Convention on Contracts for the International Sale of Goods does not apply to this Agreement. The state and federal courts of Santa Clara County, California shall have exclusive jurisdiction of any dispute arising out of this Agreement.
+
+4.7 Severability
+
+If any provision of this Agreement is held to be invalid, illegal or unenforceable, the remaining provisions shall be unaffected thereby and remain valid as if such provision had not been set forth herein.
+
+4.8 Entire Agreement
+
+This Agreement states all the terms agreed between the parties and supersedes all other agreements between the parties as of the date of acceptance relating to its subject matter.
+
+4.9 No Waiver
+
+Google will not be treated as having waived any rights by not exercising (or delaying the exercise of) any rights under this Agreement.
+
+Appendix
+
+Gemma 1
+
+Gemma 1.1
+
+Gemma 2
+
+Gemma 3
+
+Gemma 3n
+
+FunctionGemma
+
+EmbeddingGemma
+
+PaliGemma
+
+PaliGemma 2
+
+ShieldGemma
+
+ShieldGemma 2
+
+CodeGemma
+
+CodeGemma 1.1
+
+Gemma 2 JPN
+
+DataGemma RIG
+
+DataGemma RAG
+
+RecurrentGemma
+
+Gemma Scope
+
+Gemma-APS
+
+T5Gemma
+
+VaultGemma
+
+FunctionGemma
+
+T5Gemma 2
+
+TranslateGemma

package/licenses/llama.cpp-MIT.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023-2026 The ggml authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/licenses/qwen3-embedding-notice.txt

@@ -0,0 +1,13 @@
+Qwen3-Embedding-0.6B
+
+The Qwen3-Embedding-0.6B model asset is provided under the Apache License, Version 2.0. The pinned GGUF asset is published by Qwen at:
+https://huggingface.co/Qwen/Qwen3-Embedding-0.6B-GGUF
+
+Model card and pinned asset revision:
+https://huggingface.co/Qwen/Qwen3-Embedding-0.6B-GGUF/tree/370f27d7550e0def9b39c1f16d3fbaa13aa67728
+
+Apache License, Version 2.0:
+https://www.apache.org/licenses/LICENSE-2.0
+The text is also included in licenses/apache-2.0.txt.
+
+This notice identifies the model license and provenance. The model weights are not included in the npm package; release jobs fetch the immutable URL in licenses/embedding-assets.json and verify its SHA-256 before staging.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autorag/librarian",
-  "version": "2.4.2",
+  "version": "2.5.0",
   "description": "Self-evolving librarian agent on Pi framework with pluggable retrieval methods",
   "license": "MIT",
   "repository": {
@@ -26,7 +26,9 @@
   "files": [
     "dist",
     "skills",
-    "README.md"
+    "README.md",
+    "NOTICE",
+    "licenses"
   ],
   "publishConfig": {
     "access": "public"
@@ -39,18 +41,20 @@
     "test:windows": "node scripts/run-test-files.mjs",
     "check": "biome check --write .",
     "lint": "biome check .",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc --noEmit",
+    "check:embedding-manifest": "node scripts/check-embedding-manifest.mjs",
+    "supply-chain": "bun scripts/supply-chain/evaluate.ts gate --sbom sbom.local.cdx.json"
   },
   "engines": {
     "node": ">=24.0.0"
   },
   "dependencies": {
-    "@earendil-works/pi-agent-core": "^0.84.4",
-    "@earendil-works/pi-ai": "^0.84.4",
-    "@earendil-works/pi-coding-agent": "0.84.4",
-    "@earendil-works/pi-tui": "0.84.4",
+    "@earendil-works/pi-agent-core": "^0.85.1",
+    "@earendil-works/pi-ai": "^0.85.1",
+    "@earendil-works/pi-coding-agent": "0.85.1",
+    "@earendil-works/pi-tui": "0.85.1",
     "@opendataloader/pdf": "^2.5.7",
-    "@rhwp/core": "0.8.4",
+    "@rhwp/core": "0.8.6",
     "chardet": "^2.2.0",
     "fast-xml-parser": "^5.11.1",
     "iconv-lite": "^0.7.3",
@@ -67,7 +71,8 @@
     "@biomejs/biome": "2.5.12",
     "@types/mailparser": "^3.4.6",
     "@types/node": "^26.4.1",
+    "@types/ws": "^8.18.1",
     "typescript": "^7.0.2",
-    "vitest": "^4.1.11"
+    "vitest": "^5.0.0"
   }
 }

package/skills/autorag/SKILL.md

@@ -103,21 +103,27 @@ autorag feedback <sessionId> --useful 1,3 --not-useful 2 --json
 autorag status --json
 autorag health --json
 autorag refresh --json
-autorag refresh --method bm25,minsync,jikji --json
+autorag refresh --method minsync,jikji --json
 autorag watch --once --json
 autorag watch
 autorag refresh --force --json
 autorag index rebuild --yes --json
-autorag index reset --method bm25 --yes --json
+autorag index reset --method parsed --yes --json
 autorag memory inspect --json
+autorag tui
+autorag serve --force
+autorag p2p policy list --json
 ```
 
 Prefer a full refresh so parsed mirrors, MinSync, Jikji, and configured
 datasources stay aligned. `--method` accepts
-`parsed,minsync,datasources,jikji,all`. Use it only for deliberate
-narrowing. Scheduled maintenance should use non-daemon `autorag watch --once`,
-typically every 15–30 minutes, with the same config used by search and no
-overlapping runs.
+`parsed,minsync,datasources,jikji,all`. BM25 is a MinSync retrieval mode, not
+a `--method` name. Use `--method` only for deliberate narrowing. Scheduled
+maintenance should use non-daemon `autorag watch --once`, typically every
+1 hour, with the same config used by search and no overlapping runs.
+`autorag tui` is the shipped beta terminal UI. `autorag serve` / `autorag p2p`
+are opt-in SimpleX peer sharing (disabled until `p2p.enabled` is true, unless
+`--force`).
 
 Reset and rebuild commands remove only selected workspace `.autorag` indexes.
 Never target source documents. `memory inspect` is read-only and path-opaque.
@@ -131,3 +137,7 @@ Never target source documents. `memory inspect` is read-only and path-opaque.
 - Preserve real source mapping and numbered feedback identifiers.
 - Prefer `--json --debug` when another agent consumes search output or will
   call `autorag feedback`.
+- Do not invent CLI commands. `autorag --help` is the command list of record,
+  including the shipped `gateway`, `models`, `serve`, `p2p`, `tui`, and `lite`
+  commands. See `docs/embedding-runtime.md` for the local embedding runtime
+  and gateway setup.

package/skills/autorag-lite-setup/SKILL.md

@@ -77,14 +77,14 @@ setup. Configure datasources directly in trusted config, wizard-style:
    list to the user, with what is missing for each.
 4. Set up a skipped datasource only when the user explicitly asks for it:
    install or authenticate the backing CLI first, then configure it.
-5. E-mail datasources (`gmail`, `mail-export`, `mailcrawl`) matter to most
+5. E-mail datasources (`mail-export`, `mailcrawl`) matter to most
    users — always probe them and report their status, even when they end up
    skipped.
 
 Datasource skills belong in trusted config and remain default-deny.
 
 Config keys may be builtin template names (`kakao`, `whatsapp`, `telegram`,
-`slack`, `discord`, `clawgallery`, `notion`, `github`, `cloud-drive`, `gmail`,
+`slack`, `discord`, `clawgallery`, `notion`, `github`, `cloud-drive`,
 `mail-export`, `mailcrawl`, `obsidian`, `rss`, `spotlight`) or connection
 aliases with `"type": "<template>"`. Unknown names are skipped with an
 `unknown-datasource-skill` warning; they do not fail config resolution.

package/skills/autorag-setup/SKILL.md

@@ -185,13 +185,13 @@ setup. Configure datasources directly in trusted config, wizard-style:
    list to the user, with what is missing for each.
 4. Set up a skipped datasource only when the user explicitly asks for it:
    install or authenticate the backing CLI first, then configure it.
-5. E-mail datasources (`gmail`, `mail-export`, `mailcrawl`) matter to most
+5. E-mail datasources (`mail-export`, `mailcrawl`) matter to most
    users — always probe them and report their status, even when they end up
    skipped.
 
 Datasource skills belong in trusted config and remain default-deny. Builtin
 template names are `kakao`, `whatsapp`, `telegram`, `slack`, `discord`,
-`clawgallery`, `notion`, `github`, `cloud-drive`, `gmail`, `mail-export`,
+`clawgallery`, `notion`, `github`, `cloud-drive`, `mail-export`,
 `mailcrawl`, `obsidian`, `rss`, and `spotlight`. Config keys may be connection
 aliases with `"type": "<template>"`. Unknown names are skipped with an
 `unknown-datasource-skill` warning; they do not fail config resolution.
@@ -203,14 +203,13 @@ aliases with `"type": "<template>"`. Unknown names are skipped with an
     "github": { "connector": { "repos": ["owner/repo"], "tokenEnv": "GITHUB_TOKEN" } },
     "google-drive": { "type": "cloud-drive", "connector": { "provider": "google-drive", "remote": "gdrive:" } },
     "archive-drive": { "type": "cloud-drive", "connector": { "remote": "archive:" } },
-    "gmail": { "connector": { "tokenEnv": "GMAIL_ACCESS_TOKEN", "labelIds": ["INBOX"] } },
     "mailcrawl": { "instanceId": "personal", "connector": { "account": "personal", "mailbox": "INBOX", "binaryPath": "mailcrawl" } },
     "obsidian": { "connector": { "vaultPath": "/path/to/vault" } },
     "rss": { "connector": { "feeds": [{ "url": "https://example.com/feed.xml" }] } }
   },
   "datasourceAccess": {
-    "allowedTags": ["github", "cloud-drive", "gmail", "mailcrawl", "obsidian", "rss"],
-    "allowedScopes": ["/github/**", "/google-drive/**", "/archive-drive/**", "/gmail/**", "/mailcrawl/**", "/obsidian/**", "/rss/**"]
+    "allowedTags": ["github", "cloud-drive", "mailcrawl", "obsidian", "rss"],
+    "allowedScopes": ["/github/**", "/google-drive/**", "/archive-drive/**", "/mailcrawl/**", "/obsidian/**", "/rss/**"]
   }
 }
 ```
@@ -218,14 +217,13 @@ aliases with `"type": "<template>"`. Unknown names are skipped with an
 Tokens are environment-variable names, not raw secrets. CLI-backed connectors
 keep authentication in their external tool configuration.
 
-Mailcrawl must be installed separately (`@nomadamas/mailcrawl@0.1.4` or newer)
+Mailcrawl must be installed separately (`@nomadamas/mailcrawl@0.1.6` or newer)
 and configured through its own Himalaya account. AutoRAG runs its local `sync`
 and `index` lifecycle, then uses the mailcrawl CLI for BM25, semantic, or
 hybrid search. Do not use 0.1.3 or earlier: a no-op sync followed by `index`
 fails with `text array must be non-empty`.
-Use mailcrawl for Himalaya-backed IMAP/Maildir retrieval. The legacy
-`gmail` connector option `backend: "himalaya"` is no longer registered; migrate
-that configuration to an explicit `mailcrawl` datasource.
+Use mailcrawl for Gmail, IMAP, and Maildir retrieval. The former Gmail REST
+datasource is removed.
 
 ## Verify and build indexes
 

package/dist/datasource/skills/gmail/connector.d.ts

@@ -1,30 +0,0 @@
-/**
- * Gmail connector (issue #1304).
- *
- * Lists and fetches messages through the Gmail REST v1 API with a trusted,
- * server-configured OAuth access token. Per-message fetch failures degrade
- * to warnings. Never throws; messages stay path/PII-opaque (addresses appear
- * only inside indexed chunk content, never in diagnostics).
- */
-import type { ConnectorFetchResult, DatasourceConnector } from "../../connector.js";
-export interface GmailConnectorOptions {
-    /** Gmail REST base; override to point at a mock server in tests. */
-    readonly baseUrl?: string;
-    /** OAuth access token; explicit value wins over {@link tokenEnv}. */
-    readonly token?: string;
-    /** Env var name holding the access token. Default `GMAIL_ACCESS_TOKEN`. */
-    readonly tokenEnv?: string;
-    /** Restrict to these label ids (e.g. `["INBOX"]`). */
-    readonly labelIds?: readonly string[];
-    /** Gmail search query (the `q` parameter). */
-    readonly query?: string;
-    readonly timeoutMs?: number;
-    readonly fetchImpl?: typeof fetch;
-    readonly maxPages?: number;
-    readonly maxDocuments?: number;
-}
-export declare class GmailConnector implements DatasourceConnector {
-    private readonly options;
-    constructor(options?: GmailConnectorOptions);
-    fetch(signal?: AbortSignal): Promise<ConnectorFetchResult>;
-}

package/dist/datasource/skills/gmail/index.d.ts

@@ -1,2 +0,0 @@
-export { GmailConnector, type GmailConnectorOptions } from "./connector.js";
-export { GMAIL_SKILL_DEFINITION, GmailSkill, type GmailSkillOptions } from "./skill.js";

package/dist/datasource/skills/gmail/skill.d.ts

@@ -1,24 +0,0 @@
-/**
- * Gmail datasource skill (issue #1304).
- *
- * Thin composition of GmailConnector with the shared
- * {@link ConnectorDatasourceSkill} base: descriptor, polling metadata,
- * ok/fail indexing with path/PII-opaque diagnostics, lexical retrieval via
- * the shared pipeline, opaque `/gmail/<instance>/…` sources, and a
- * progressive-disclosure agent-skill manifest.
- */
-import type { DatasourceConnector } from "../../connector.js";
-import { ConnectorDatasourceSkill, type ConnectorSkillDefinition, type ConnectorSkillOptions } from "../../connector-skill.js";
-import { type GmailConnectorOptions } from "./connector.js";
-export declare const GMAIL_SKILL_DEFINITION: ConnectorSkillDefinition;
-export interface GmailSkillOptions extends Omit<ConnectorSkillOptions, "connector"> {
-    /**
-     * Trusted pre-built connector; wins over {@link connectorOptions}. Accepts
-     * any compatible mail-backed connector.
-     */
-    readonly connector?: DatasourceConnector;
-    readonly connectorOptions?: GmailConnectorOptions;
-}
-export declare class GmailSkill extends ConnectorDatasourceSkill {
-    constructor(options?: GmailSkillOptions);
-}