079project 8.0.0 → 9.1.0

package/LICENSE

@@ -0,0 +1,165 @@
+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

package/README.en.md

@@ -55,7 +55,7 @@ Default ports:
 
 - Main AI: `127.0.0.1:5080`
 - Identity + Web: `127.0.0.1:5081`
-
+## Attention! before all mode start,you must start a redis server first.
 ### Mode A (Recommended): Identity+Web hosts UI and proxies `/api`
 
 1) Start the main AI process (serves `/api/*` only):
@@ -67,6 +67,12 @@ $env:AI_AUTH_JWT_SECRET="change-me"
 node .\main.cjs
 ```
 
+If you want to use the “performance/concurrency experimental switches” (`--group-proc` / `--infer-mp` / `--single-proc`), use:
+
+```powershell
+node .\mainFailedOfJing1Xi4Hua4Zhi4Duan3Yu3.cjs
+```
+
 2) Start the identity + web process (hosts UI and proxies `/api/*` to the main AI):
 
 ```powershell
@@ -127,6 +133,72 @@ Open: `http://127.0.0.1:5081/`
 
 ---
 
+## Quick Start Matrix (By Scenario)
+
+Notes:
+
+- `main.cjs`: “classic single-process” entry (stable/default).
+- `mainFailedOfJing1Xi4Hua4Zhi4Duan3Yu3.cjs`: entry that includes experimental concurrency switches (`--group-proc` / `--infer-mp` / `--single-proc`).
+
+### Scenario 1: Minimal local dev (single process + no auth + fast boot)
+
+```powershell
+cd "079Project/"
+$env:AI_AUTH_ENABLED="false"
+$env:AI_ROBOTS_AUTOLOAD="false"
+node .\mainFailedOfJing1Xi4Hua4Zhi4Duan3Yu3.cjs --single-proc=true --robots-autoload=false --tests-autoload=false --port=5080
+```
+
+### Scenario 2: Recommended daily use (two processes: Identity/UI + API reverse proxy)
+
+Main AI:
+
+```powershell
+cd "079Project/"
+$env:AI_AUTH_ENABLED="true"
+$env:AI_AUTH_JWT_SECRET="change-me"
+node .\main.cjs --port=5080
+```
+
+Identity + Web:
+
+```powershell
+cd "079Project/"
+$env:AUTH_JWT_SECRET="change-me"
+$env:AI_API_BASE="http://127.0.0.1:5080"
+node .\auth_frontend_server.cjs
+```
+
+### Scenario 3: Force single-process (more predictable if multi-process is slower)
+
+```powershell
+cd "079Project/"
+node .\mainFailedOfJing1Xi4Hua4Zhi4Duan3Yu3.cjs --single-proc=true --port=5080
+```
+
+`--single-proc=true` forces `--group-proc=false` and `--infer-mp=false`.
+
+### Scenario 4: Higher throughput on multi-core (one process per group)
+
+Good for heavier requests and higher concurrency; may be slower if IPC overhead dominates.
+
+```powershell
+cd "079Project/"
+$env:AI_AUTH_ENABLED="false"
+node .\mainFailedOfJing1Xi4Hua4Zhi4Duan3Yu3.cjs --group-proc=true --group-count=6 --group-size=3 --port=5080
+```
+
+### Scenario 5: Multi-core inference pool (Spark dispatch only; not group-proc)
+
+```powershell
+cd "079Project/"
+node .\mainFailedOfJing1Xi4Hua4Zhi4Duan3Yu3.cjs --infer-mp=true --infer-workers=6 --port=5080
+```
+
+Note: when `group-proc` is enabled, `infer-mp` is automatically disabled (to avoid double multi-processing).
+
+---
+
 ## Main AI CLI Options (Common)
 
 `main.cjs` supports some CLI / env options (partial list):
@@ -145,6 +217,14 @@ Example: listen on `0.0.0.0:5080`:
 node .\main.cjs --gateway-host=0.0.0.0 --port=5080
 ```
 
+Extra options supported by `mainFailedOfJing1Xi4Hua4Zhi4Duan3Yu3.cjs` (concurrency/perf experiments):
+
+- `--single-proc` / `AI_SINGLE_PROC`: force single-process (disables `group-proc` and `infer-mp`).
+- `--group-proc` / `AI_GROUP_PROC`: one process per group.
+- `--group-proc-timeout-ms` / `AI_GROUP_PROC_TIMEOUT_MS`: RPC timeout for group workers.
+- `--infer-mp` / `AI_INFER_MP`: inference worker pool (Spark dispatch only).
+- `--infer-workers` / `AI_INFER_WORKERS`: number of inference workers.
+
 ---
 
 ## Identity System (SQLite + JWT)

package/README.md

@@ -55,7 +55,7 @@ npm install
 
 - 主 AI：`127.0.0.1:5080`
 - 身份 + 前端：`127.0.0.1:5081`
-
+## 注意,在所有的启动之前,必须要先启动一个redis服务器
 ### 模式 A：推荐（身份+前端托管 + /api 反代）
 
 1) 启动主 AI（只提供 `/api/*`）：
@@ -67,6 +67,12 @@ $env:AI_AUTH_JWT_SECRET="change-me"
 node .\main.cjs
 ```
 
+如果你需要启用“性能/并发实验功能”（`--group-proc`/`--infer-mp`/`--single-proc`），请改用：
+
+```powershell
+node .\mainFailedOfJing1Xi4Hua4Zhi4Duan3Yu3.cjs
+```
+
 2) 启动身份服务 + 前端托管（同时反代 `/api/*` 到主 AI）：
 
 ```powershell
@@ -129,6 +135,76 @@ node .\auth_frontend_server.cjs
 
 ---
 
+## 启动方式速查（按场景）
+
+说明：
+
+- `main.cjs`：偏“传统单进程”入口（稳定/默认）。
+- `mainFailedOfJing1Xi4Hua4Zhi4Duan3Yu3.cjs`：包含并发实验开关（`--group-proc`/`--infer-mp`/`--single-proc`）的入口。
+
+### 场景 1：最简本地调试（单进程 + 免鉴权 + 快速启动）
+
+```powershell
+cd "079Project/"
+$env:AI_AUTH_ENABLED="false"
+$env:AI_ROBOTS_AUTOLOAD="false"
+node .\mainFailedOfJing1Xi4Hua4Zhi4Duan3Yu3.cjs --single-proc=true --robots-autoload=false --tests-autoload=false --port=5080
+```
+
+### 场景 2：推荐日常使用（两进程：身份/前端托管 + API 反代）
+
+主 AI：
+
+```powershell
+cd "079Project/"
+$env:AI_AUTH_ENABLED="true"
+$env:AI_AUTH_JWT_SECRET="change-me"
+node .\main.cjs --port=5080
+```
+
+身份 + 前端：
+
+```powershell
+cd "079Project/"
+$env:AUTH_JWT_SECRET="change-me"
+$env:AI_API_BASE="http://127.0.0.1:5080"
+node .\auth_frontend_server.cjs
+```
+
+### 场景 3：强制单进程（避免多进程导致变慢/更稳定）
+
+当你发现多进程模式在当前机器/当前负载下反而更慢，使用：
+
+```powershell
+cd "079Project/"
+node .\mainFailedOfJing1Xi4Hua4Zhi4Duan3Yu3.cjs --single-proc=true --port=5080
+```
+
+说明：`--single-proc=true` 会强制关闭 `--group-proc` 与 `--infer-mp`。
+
+### 场景 4：多核吞吐（每个 group 一个进程）
+
+适用于并发请求较多、单次请求较重、且 CPU 核数充足的机器；如果请求很轻或 IPC 开销占比过高，可能“更慢”。
+
+```powershell
+cd "079Project/"
+$env:AI_AUTH_ENABLED="false"
+node .\mainFailedOfJing1Xi4Hua4Zhi4Duan3Yu3.cjs --group-proc=true --group-count=6 --group-size=3 --port=5080
+```
+
+### 场景 5：多核加速（推理多进程池，仅用于 Spark 汇聚的 respond 分发）
+
+适用于不启用 `group-proc` 的情况下，想让 SparkArray/BigSparkArray 的分发利用多核：
+
+```powershell
+cd "079Project/"
+node .\mainFailedOfJing1Xi4Hua4Zhi4Duan3Yu3.cjs --infer-mp=true --infer-workers=6 --port=5080
+```
+
+说明：启用 `group-proc` 时会自动禁用 `infer-mp`（避免双层多进程）。
+
+---
+
 ## 后端启动参数（常用）
 
 `main.cjs` 支持部分 CLI/环境变量（这里只列近期常用的）：
@@ -147,6 +223,14 @@ node .\auth_frontend_server.cjs
 node .\main.cjs --gateway-host=0.0.0.0 --port=5080
 ```
 
+`mainFailedOfJing1Xi4Hua4Zhi4Duan3Yu3.cjs` 额外支持（并发/性能实验）：
+
+- `--single-proc` / `AI_SINGLE_PROC`：强制单进程（关闭 `group-proc` 与 `infer-mp`）。
+- `--group-proc` / `AI_GROUP_PROC`：每个 group 一个进程。
+- `--group-proc-timeout-ms` / `AI_GROUP_PROC_TIMEOUT_MS`：group RPC 超时。
+- `--infer-mp` / `AI_INFER_MP`：推理多进程池（仅分发 Spark 汇聚推理）。
+- `--infer-workers` / `AI_INFER_WORKERS`：推理子进程数量。
+
 ## 身份系统（SQLite + JWT）
 
 身份系统存储：默认 `079Project/runtime_store/auth.sqlite`。

package/groupWorker.cjs

@@ -0,0 +1,253 @@
+'use strict';
+
+const path = require('path');
+
+const GROUP_ID = String(process.env.AI_GROUP_ID || 'G1').trim() || 'G1';
+const GROUP_SIZE = Math.max(1, Number(process.env.AI_GROUP_SIZE || 7) || 7);
+const LMDB_ROOT = String(process.env.AI_LMDB_ROOT || path.join(__dirname, 'lmdb'));
+const LMDB_MAP_BYTES = Math.max(64 * 1024 * 1024, Number(process.env.AI_LMDB_MAP_BYTES || 512 * 1024 * 1024) || 512 * 1024 * 1024);
+
+const core = require('./mainFailedOfJing1Xi4Hua4Zhi4Duan3Yu3.cjs');
+
+const { LmdbStore, NamespacedStore, RuntimeState, MemeBarrier } = core;
+const modelDefaults = core.MODEL_DEFAULTS || {};
+
+const safeCall = async (fn) => {
+  try {
+    const result = await Promise.resolve().then(fn);
+    return { ok: true, result };
+  } catch (e) {
+    return { ok: false, error: e?.message || String(e) };
+  }
+};
+
+const kvmStore = new LmdbStore({ name: 'kvm', rootDir: LMDB_ROOT, mapSizeBytes: LMDB_MAP_BYTES });
+const memeStore = new LmdbStore({ name: 'meme_graph', rootDir: LMDB_ROOT, mapSizeBytes: LMDB_MAP_BYTES });
+const sessionStore = new LmdbStore({ name: 'session', rootDir: LMDB_ROOT, mapSizeBytes: LMDB_MAP_BYTES });
+
+const sharedStores = {
+  kvmStore: new NamespacedStore(kvmStore, `${GROUP_ID}:kvm`),
+  memeStore: new NamespacedStore(memeStore, `${GROUP_ID}:graph`),
+  sessionStore: new NamespacedStore(sessionStore, `${GROUP_ID}:session`)
+};
+
+const controllers = {}; // name -> { runtime }
+const controllerNames = [];
+for (let i = 0; i < GROUP_SIZE; i++) {
+  const name = `${GROUP_ID}_AI${i + 1}`;
+  controllerNames.push(name);
+  const stores = {
+    ...sharedStores,
+    config: {
+      controllerName: name,
+      groupId: GROUP_ID,
+      groupIndex: i + 1,
+      robotsDir: process.env.AI_ROBOTS_DIR || undefined,
+      testsDir: process.env.AI_TESTS_DIR || undefined
+    }
+  };
+  controllers[name] = { runtime: new RuntimeState(stores) };
+}
+
+const getRuntime = (controllerName) => {
+  const name = String(controllerName || '').trim();
+  if (!name) throw new Error('controllerName required');
+  const entry = controllers[name];
+  if (!entry) throw new Error(`controller-not-found:${name}`);
+  return entry.runtime;
+};
+
+const allowedRuntimeCalls = new Set([
+  'onlineLookup',
+  'collectRobotsDocuments',
+  'listRobotsFiles',
+  'exportGraphToFile',
+  'getSearchConfig',
+  'setSearchConfig',
+  'learnFromDialog'
+]);
+
+let memeBarrier = null;
+const getBarrier = (maliciousThreshold) => {
+  if (!MemeBarrier) {
+    throw new Error('MemeBarrier export missing');
+  }
+  if (!memeBarrier) {
+    const runtime = controllers[controllerNames[0]]?.runtime;
+    if (!runtime) {
+      throw new Error('no-runtime-for-barrier');
+    }
+    memeBarrier = new MemeBarrier(runtime, {
+      maliciousThreshold: Number.isFinite(Number(maliciousThreshold))
+        ? Number(maliciousThreshold)
+        : (modelDefaults.maliciousThreshold ?? 0.7)
+    });
+  }
+  return memeBarrier;
+};
+
+process.on('message', async (msg) => {
+  if (!msg || typeof msg !== 'object') return;
+  const { id, cmd } = msg;
+  if (!id || !cmd) return;
+
+  const reply = await safeCall(async () => {
+    if (cmd === 'respond') {
+      const runtime = getRuntime(msg.controllerName);
+      return runtime.processInput(msg.payload || {});
+    }
+
+    if (cmd === 'metrics') {
+      const runtime = getRuntime(msg.controllerName);
+      return {
+        name: String(msg.controllerName),
+        online: true,
+        health: { status: 'ok', since: Date.now(), failures: 0 },
+        metrics: runtime.metrics,
+        params: runtime.cloneParams()
+      };
+    }
+
+    if (cmd === 'cloneParams') {
+      const runtime = getRuntime(msg.controllerName);
+      return runtime.cloneParams();
+    }
+
+    if (cmd === 'applyParams') {
+      const runtime = getRuntime(msg.controllerName);
+      runtime.setParams(msg.params || {});
+      return true;
+    }
+
+    if (cmd === 'snapshot') {
+      const runtime = getRuntime(msg.controllerName);
+      return runtime.toSnapshot();
+    }
+
+    if (cmd === 'applySnapshot') {
+      const runtime = getRuntime(msg.controllerName);
+      await runtime.fromSnapshot(msg.snapshot);
+      return true;
+    }
+
+    if (cmd === 'applySnapshotAll') {
+      const snapshot = msg.snapshot;
+      for (const name of controllerNames) {
+        const runtime = getRuntime(name);
+        await runtime.fromSnapshot(snapshot);
+      }
+      return true;
+    }
+
+    if (cmd === 'listMetrics') {
+      return controllerNames.map((name) => {
+        const runtime = controllers[name].runtime;
+        return {
+          name,
+          online: true,
+          health: { status: 'ok', since: Date.now(), failures: 0 },
+          metrics: runtime.metrics,
+          params: runtime.cloneParams()
+        };
+      });
+    }
+
+    if (cmd === 'ingestDocument') {
+      const doc = msg.doc || {};
+      const results = [];
+      for (const name of controllerNames) {
+        const runtime = getRuntime(name);
+        results.push(await runtime.ingestDocument(doc));
+      }
+      return results;
+    }
+
+    if (cmd === 'ingestDocumentTo') {
+      const runtime = getRuntime(msg.controllerName);
+      return runtime.ingestDocument(msg.doc || {});
+    }
+
+    if (cmd === 'ingestDocumentToGroup') {
+      const doc = msg.doc || {};
+      const results = [];
+      for (const name of controllerNames) {
+        const runtime = getRuntime(name);
+        results.push(await runtime.ingestDocument(doc));
+      }
+      return results;
+    }
+
+    if (cmd === 'forgetMemes') {
+      const criteria = msg.criteria || {};
+      const results = [];
+      for (const name of controllerNames) {
+        const runtime = getRuntime(name);
+        results.push(await runtime.forgetMemes(criteria));
+      }
+      return results;
+    }
+
+    if (cmd === 'runtimeCall') {
+      const runtime = getRuntime(msg.controllerName);
+      const method = String(msg.method || '').trim();
+      if (!allowedRuntimeCalls.has(method)) {
+        throw new Error(`runtimeCall not allowed: ${method}`);
+      }
+      const args = Array.isArray(msg.args) ? msg.args : [];
+      const fn = runtime[method];
+      if (typeof fn !== 'function') {
+        throw new Error(`runtime method missing: ${method}`);
+      }
+      return fn.apply(runtime, args);
+    }
+
+    if (cmd === 'memebarrierStart') {
+      const barrier = getBarrier(msg.maliciousThreshold);
+      if (typeof msg.maliciousThreshold === 'number' && Number.isFinite(msg.maliciousThreshold)) {
+        barrier.maliciousThreshold = msg.maliciousThreshold;
+      }
+      barrier.start();
+      return { running: true, threshold: barrier.maliciousThreshold, stats: barrier.getStats() };
+    }
+
+    if (cmd === 'memebarrierStop') {
+      if (memeBarrier) {
+        memeBarrier.stop();
+      }
+      return { running: false };
+    }
+
+    if (cmd === 'memebarrierStats') {
+      if (!memeBarrier) {
+        return { running: false, threshold: (modelDefaults.maliciousThreshold ?? 0.7), stats: null };
+      }
+      return { running: Boolean(memeBarrier.running), threshold: memeBarrier.maliciousThreshold, stats: memeBarrier.getStats() };
+    }
+
+    if (cmd === 'memebarrierSetThreshold') {
+      const value = Number(msg.maliciousThreshold);
+      if (!Number.isFinite(value)) {
+        throw new Error('invalid-threshold');
+      }
+      const barrier = getBarrier(value);
+      barrier.maliciousThreshold = value;
+      return { threshold: barrier.maliciousThreshold };
+    }
+
+    throw new Error(`unknown-cmd:${cmd}`);
+  });
+
+  try {
+    process.send({ id, ...reply });
+  } catch (_e) {
+    // ignore
+  }
+});
+
+process.on('SIGINT', () => {
+  process.exit(0);
+});
+
+process.on('disconnect', () => {
+  process.exit(0);
+});