079project 1.0.0

package/groupmanager.cjs

@@ -0,0 +1,143 @@
+const fs = require('fs');
+const path = require('path');
+
+// 轻量参数解析
+function parseArgs(argv) {
+  const out = {};
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a.startsWith('--')) {
+      const key = a.slice(2);
+      const val = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : true;
+      out[key] = val;
+    }
+  }
+  return out;
+}
+
+const args = parseArgs(process.argv.slice(2));
+const SOURCE_ROOT = path.resolve(args['source-root'] || __dirname);
+const TARGET_GROUPS_DIR = path.resolve(args['groups-dir'] || path.join(__dirname, 'groups'));
+const ROBOTS_DIR = path.resolve(args['robots-dir'] || path.join(SOURCE_ROOT, 'robots'));
+const TESTS_DIR = path.resolve(args['tests-dir'] || path.join(SOURCE_ROOT, 'tests'));
+const PUBLIC_DIR = path.resolve(args['public-dir'] || path.join(SOURCE_ROOT, 'public'));
+const SNAPSHOTS_DIR = path.resolve(args['snapshots-dir'] || path.join(SOURCE_ROOT, 'snapshots'));
+const MAX_SIZE_KB = Number(args['shard-size-kb'] || 250); // 默认每分片约250KB
+const CLEAN = Boolean(args['clean'] && String(args['clean']).toLowerCase() !== 'false');
+
+console.log('[CFG] source-root =', SOURCE_ROOT);
+console.log('[CFG] robots-dir  =', ROBOTS_DIR);
+console.log('[CFG] tests-dir   =', TESTS_DIR);
+console.log('[CFG] public-dir  =', PUBLIC_DIR);
+console.log('[CFG] snapshots   =', SNAPSHOTS_DIR);
+console.log('[CFG] groups-dir  =', TARGET_GROUPS_DIR);
+console.log('[CFG] shard-size  =', MAX_SIZE_KB, 'KB');
+
+// 必要文件和文件夹
+const REQUIRED_FILES = [
+  'main_Serve.cjs',
+  'main_Study.cjs',
+  'forwarder.js',
+  'controller.cjs',
+  'model_RNN.py',
+  'propagateSignalUseJsWroker.js',
+  'memeMergeWorker.cjs',
+  'boot.py',
+  'model_RNN.py',
+  'runtime.proto',
+  'lemma.csv'
+];
+const REQUIRED_DIRS = [
+  'robots',
+  'tests',
+  'public',
+  'snapshots'
+];
+
+// 检查与准备目标 groups 文件夹
+if (!fs.existsSync(TARGET_GROUPS_DIR)) {
+  fs.mkdirSync(TARGET_GROUPS_DIR, { recursive: true });
+  console.log('已创建 groups 文件夹:', TARGET_GROUPS_DIR);
+} else if (CLEAN) {
+  // 清理旧分组
+  for (const f of fs.readdirSync(TARGET_GROUPS_DIR)) {
+    if (/^group_\d+$/.test(f)) {
+      const p = path.join(TARGET_GROUPS_DIR, f);
+      try { fs.rmSync(p, { recursive: true, force: true }); } catch {}
+    }
+  }
+  console.log('已清理旧分片 group_*');
+}
+
+// 读取 robots 目录下所有 .txt 文件并拼接内容
+if (!fs.existsSync(ROBOTS_DIR)) {
+  console.error('[ERROR] robots 目录不存在:', ROBOTS_DIR);
+  process.exit(1);
+}
+const robotFiles = fs.readdirSync(ROBOTS_DIR).filter(f => f.toLowerCase().endsWith('.txt'));
+if (robotFiles.length === 0) {
+  console.warn('[WARN] robots 目录未发现 .txt 文件:', ROBOTS_DIR);
+}
+let allLines = [];
+for (const file of robotFiles) {
+  const filePath = path.join(ROBOTS_DIR, file);
+  const lines = fs.readFileSync(filePath, 'utf-8')
+    .split(/\r?\n/)
+    .filter(line => line.length > 0);
+  for (const line of lines) allLines.push(line);
+}
+
+// 按指定KB分割，每行1句
+const MAX_SIZE = MAX_SIZE_KB * 1024;
+let shards = [];
+let currentShard = [];
+let currentSize = 0;
+for (const line of allLines) {
+  const lineSize = Buffer.byteLength(line, 'utf-8') + 1; // +1 for newline
+  if (currentSize + lineSize > MAX_SIZE && currentShard.length > 0) {
+    shards.push(currentShard);
+    currentShard = [line];
+    currentSize = lineSize;
+  } else {
+    currentShard.push(line);
+    currentSize += lineSize;
+  }
+}
+if (currentShard.length > 0) shards.push(currentShard);
+
+console.log(`共分割为 ${shards.length} 个 group，每组约${MAX_SIZE_KB}KB`);
+
+// 复制文件和文件夹到每个 group_${x} 文件夹，并只包含该分片的 robots
+for (let i = 0; i < shards.length; i++) {
+  const groupFolder = path.join(TARGET_GROUPS_DIR, `group_${i}`);
+  if (!fs.existsSync(groupFolder)) fs.mkdirSync(groupFolder, { recursive: true });
+
+  // 复制必要文件（从 SOURCE_ROOT 获取）
+  for (const file of REQUIRED_FILES) {
+    const src = path.join(SOURCE_ROOT, file);
+    const dest = path.join(groupFolder, file);
+    if (fs.existsSync(src)) { try { fs.copyFileSync(src, dest); } catch {} }
+  }
+
+  // 复制必要文件夹（按参数映射）
+  const DIR_MAP = { robots: ROBOTS_DIR, tests: TESTS_DIR, public: PUBLIC_DIR, snapshots: SNAPSHOTS_DIR };
+  for (const dir of REQUIRED_DIRS) {
+    const srcDir = DIR_MAP[dir] || path.join(SOURCE_ROOT, dir);
+    const destDir = path.join(groupFolder, dir);
+    if (!fs.existsSync(srcDir)) continue;
+    if (!fs.existsSync(destDir)) fs.mkdirSync(destDir, { recursive: true });
+    if (dir === 'robots') {
+      const robotsTxt = path.join(destDir, `group_${i}.txt`);
+      try { fs.writeFileSync(robotsTxt, shards[i].join('\n')); } catch {}
+    } else {
+      const files = fs.readdirSync(srcDir);
+      for (const f of files) {
+        const srcFile = path.join(srcDir, f);
+        const destFile = path.join(destDir, f);
+        try { if (fs.statSync(srcFile).isFile()) fs.copyFileSync(srcFile, destFile); } catch {}
+      }
+    }
+  }
+  console.log(`已创建并初始化 ${groupFolder}`);
+}
+console.log('所有 group 初始化完成');

package/howToStart.txt

@@ -0,0 +1,8 @@
+如何使用这个AI？
+第一步，打开anaconda powershell prompt
+第二步，在anaconda powershell prompt下输入如下指令
+```powershell prompt
+cd D:/079/v0.1
+python boot.py
+在5个窗口的其中三个显示main loop ended的时候，可以打开localhost:8642进行服务
+```

package/lemma.csv

@@ -0,0 +1,210 @@
+abandon,abandons,abandoned,abandoned,abandoning
+accept,accepts,accepted,accepted,accepting
+accompany,accompanies,accompanied,accompanied,accompanying
+achieve,achieves,achieved,achieved,achieving
+act,acts,acted,acted,acting
+add,adds,added,added,adding
+admire,admires,admired,admired,admiring
+admit,admits,admitted,admitted,admitting
+advise,advises,advised,advised,advising
+afford,affords,afforded,afforded,affording
+agree,agrees,agreed,agreed,agreeing
+allow,allows,allowed,allowed,allowing
+appear,appears,appeared,appeared,appearing
+applaud,applauds,applauded,applauded,applauding
+apply,applies,applied,applied,applying
+approach,approaches,approached,approached,approaching
+approve,approves,approved,approved,approving
+argue,argues,argued,argued,arguing
+arrange,arranges,arranged,arranged,arranging
+arrest,arrests,arrested,arrested,arresting
+ask,asks,asked,asked,asking
+attach,attaches,attached,attached,attaching
+attack,attacks,attacked,attacked,attacking
+attempt,attempts,attempted,attempted,attempting
+attend,attends,attended,attended,attending
+attract,attracts,attracted,attracted,attracting
+avoid,avoids,avoided,avoided,avoiding
+be,is,was,been,being
+be,are,were,been,being
+be,am,was,been,being
+bear,bears,bore,borne,bearing
+beat,beats,beat,beaten,beating
+become,becomes,became,become,becoming
+begin,begins,began,begun,beginning
+behave,behaves,behaved,behaved,behaving
+belong,belongs,belonged,belonged,belonging
+bend,bends,bent,bent,bending
+bet,bets,bet,bet,betting
+bid,bids,bid,bid,bidding
+bind,binds,bound,bound,binding
+bite,bites,bite,bitten,biting
+bleed,bleeds,bleed,bled,bleeding
+blow,blows,blew,blown,blowing
+break,breaks,broke,broken,breaking
+breed,breeds,bred,bred,breeding
+bring,brings,brought,brought,bringing
+broadcast,broadcasts,broadcast,broadcast,broadcasting
+build,builds,built,built,building
+burn,burns,burnt/burned,burnt/burned,burning
+burst,bursts,burst,burst,bursting
+buy,buys,bought,bought,buying
+call,calls,call,call,called
+catch,catches,caught,caught,catching
+cause,causes,caused,caused,causing
+change,changes,changed,changed,changing
+charge,charges,charged,charged,charging
+check,checks,checked,checked,checking
+choose,chooses,chose,chosen,choosing
+circle,circles,circled,circled,circling
+claim,claims,claimed,claimed,claiming
+clap,claps,clapped,clapped,clapping
+clean,cleans,cleaned,cleaned,cleaning
+clear,clears,cleared,cleared,clearing
+climb,climbs,climbed,climbed,climbing
+close,closes,closed,closed,closing
+collect,collects,collected,collected,collecting
+come,comes,came,come,coming
+cost,costs,cost,cost,costing
+count,counts,counted,counted,counting
+cover,covers,covered,covered,covering
+crash,crashes,crashed,crashed,crashing
+creep,creeps,crept,crept,creeping
+cut,cuts,cut,cut,cutting
+dance,dances,danced,danced,dancing
+deal,deals,dealt,dealt,dealing
+decide,decides,decided,decided,deciding
+declare,declares,declared,declared,declaring
+decorate,decorates,decorated,decorated,decorating
+delay,delay,delayed,delayed,delaying
+depend,depends,depended,depended,depending
+describe,describes,described,described,describing
+destroy,destroys,destroyed,destroyed,destroying
+develop,develops,developed,developed,developing
+die,dies,died,died,dying
+disagree,disagrees,disagreed,disagreed,disagreeing
+disappear,disappears,disappeared,disappeared,disappearing
+discover,discover,discovered,discovered,discovering
+discuss,discusses,discussed,discussed,discussing
+divide,divides,divided,divided,dividing
+do,does,did,done,doing
+draw,draws,drew,drawn,drawing
+dream,dreams,dreamt/dreamed,dreamt/dreamed,dreaming
+drink,drinks,drank,drunk,drinking
+drive,drives,drove,driven,driving
+drop,drops,dropped,dropped,dropping
+dry,dries,dried,dried,drying
+eat,eats,ate,eaten,eating
+fall,fall,fall,fallen,falling
+feel,feels,felt,felt,feeling
+fight,fights,fought,fought,fighting
+find,finds,found,found,finding
+finish,finishes,finished,finished,finishing
+fish,fishes,fished,fished,fishing
+fit,fit,fit,fit,fitting
+fly,flies,flew,flown,flying
+forget,forgets,forgot,forgotten,forgetting
+forgive,forgives,forgave,forgiven,forgiving
+freeze,freezes,froze,frozen,freezing
+get,gets,got,got,getting
+give,gives,gave,given,giving
+go,goes,went,gone,going
+grow,grows,grew,grown,growing
+hang,hangs,hung,hung,hanging
+have,has,had,had,having
+hear,hears,heard,heard,hearing
+hide,hides,hide,hid,hidden,hiding
+hit,hits,hit,hit,hitting
+hold,holds,held,held,holding
+hurt,hurts,hurt,hurt,hurting
+keep,keeps,kept,kept,keeping
+kick,kicks,kicked,kicked,kicking
+kill,kills,killed,killed,killing
+kiss,kisses,kissed,kissed,kissing
+know,knows,knew,known,knowing
+lay,lays,laid,laid,laying
+lead,leads,led,led,leading
+learn,learns,learned/learnt,learned/learnt,learning
+leave,leaves,left,left,leaving
+lend,lends,lent,lent,lending
+let,lets,let,let,letting
+lie,lies,lay,lain,lying
+light,lights,lit/lighted,lit/lighted,lighting
+like,likes,liked,liked,liking
+lose,loses,lost,lost,losing
+make,makes,made,made,making
+mean,means,meant,meant,meaning
+meet,meets,met,met,meeting
+miss,misses,missed,missed,missing
+move,moves,moved,moved,moving
+need,needs,needed,needed,needing
+paint,paints,painted,painted,painting
+pay,pays,paid,paid,paying
+play,plays,played,played,playing
+prefer,prefers,preferred,preferred,preferring
+prepare,prepares,prepared,prepared,preparing
+present,presents,presented,presented,presenting
+print,prints,printed,printed,printing
+produce,produces,produced,produced,producing
+promise,promises,promised,promised,promising
+put,puts,put,put,putting
+read,reads,read,read,reading
+realize,realizes,realized,realized,realizing
+receive,receives,received,received,receiving
+recognize,recognizes,recognized,recognized,recognizing
+remember,remembers,remembered,remembered,remembering
+repeat,repeats,repeated,repeated,repeating
+replace,replaces,replaced,replaced,replacing
+report,reports,reported,reported,reporting
+request,requests,requested,requested,requesting
+rest,rests,rested,rested,resting
+return,returns,returned,returned,returning
+ring,rings,rang,rung,ringing
+rise,rises,rose,risen,rising
+run,runs,ran,run,running
+say,says,said,said,saying
+see,sees,saw,seen,seeing
+seek,seeks,sought,sought,seeking
+sell,sells,sold,sold,selling
+send,sends,sent,sent,sending
+set,sets,set,set,setting
+settle,settle,settled,settled,settling
+shake,shakes,shook,shaken,shaking
+shine,shines,shone,shone,shining
+shoot,shoots,shot,shot,shooting
+show,shows,showed,shown/showed,showing
+shut,shuts,shut,shut,shutting
+sing,sings,sang,sung,singing
+sink,sinks,sank,sunk,sinking
+sit,sits,sat,sat,sitting
+sleep,sleeps,slept,slept,sleeping
+slip,slips,slipped,slipped,slipping
+smell,smells,smelt/smelled,smelt/smelled,smelling
+speak,speaks,spoke,spoken,speaking
+spend,spends,spent,spent,spending
+stand,stands,stood,stood,standing
+start,starts,started,started,starting
+stay,stays,stayed,stayed,staying
+steal,steals,stole,stolen,stealing
+stick,sticks,stick,stuck,stuck,stick
+stick,sticks,stuck,stuck,stick
+stop,stops,stopped,stopped,stop
+study,studies,studied,studied,studying
+succeed,succeeds,succeeded,succeeded,succeeding
+suffer,suffers,suffered,suffered,suffering
+suggest,suggests,suggested,suggested,suggesting
+take,takes,took,taken,taking
+teach,teaches,taught,taught,teaching
+tear,tears,tore,torn,tearing
+tell,tells,told,told,telling
+think,thinks,thought,thought,thinking
+throw,throws,threw,thrown,throwing
+understand,understands,understood,understood,understanding
+wake,wakes,woke,woken,waking
+wear,wears,wore,worn,wearing
+win,wins,won,won,winning
+write,writes,wrote,written,writing
+write,writes,wrote,written,writing
+yawn,yawns,yawned,yawned,yawning
+zip,zips,zipped,zipped,zipping
+zoom,zooms,zoomed,zoomed,zooming

package/load.py

@@ -0,0 +1,35 @@
+import os
+import pandas as pd
+
+input_dir = "wikitext/wikitext-103-raw-v1"
+output_file = "robots/wikitext-103-all.txt"
+max_articles = None  # 可设为整数限制导出数量
+
+os.makedirs("robots", exist_ok=True)
+
+file_list = [
+    "train-00000-of-00002.parquet",
+    "train-00001-of-00002.parquet",
+    "validation-00000-of-00001.parquet",
+    "test-00000-of-00001.parquet"
+]
+
+idx = 0
+with open(output_file, "w", encoding="utf-8") as fout:
+    for fname in file_list:
+        fpath = os.path.join(input_dir, fname)
+        if not os.path.exists(fpath):
+            continue
+        df = pd.read_parquet(fpath)
+        for _, row in df.iterrows():
+            if max_articles and idx >= max_articles:
+                break
+            text = str(row["text"]).replace('\n\n', '\n').strip()
+            if not text:
+                continue
+            fout.write(text + "\n\n")  # 用空行分隔
+            idx += 1
+            if idx % 10000 == 0:
+                print(f"已导出 {idx} 篇...")
+
+print("导出完成！")

package/mainManager.cjs

@@ -0,0 +1,81 @@
+const { spawn } = require('child_process');
+const path = require('path');
+const fs = require('fs');
+
+// 本脚本用于“划分大组”：基于不同数据源目录，调用 groupmanager.cjs 生成
+// groups_text / groups_image / groups_audio 三个大组。
+
+function parseArgs(argv) {
+  const out = {};
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a.startsWith('--')) {
+      const key = a.slice(2);
+      const val = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : true;
+      out[key] = val;
+    }
+  }
+  return out;
+}
+
+const args = parseArgs(process.argv.slice(2));
+
+// 可单独为每个大组指定源目录；未指定则回退到工程内默认目录
+const SOURCES = {
+  text: path.resolve(args['text-root'] || path.join(__dirname)),
+  image: path.resolve(args['image-root'] || path.join(__dirname)),
+  audio: path.resolve(args['audio-root'] || path.join(__dirname)),
+};
+
+const OUTPUTS = {
+  text: path.resolve(args['text-groups'] || path.join(__dirname, 'groups_text')),
+  image: path.resolve(args['image-groups'] || path.join(__dirname, 'groups_image')),
+  audio: path.resolve(args['audio-groups'] || path.join(__dirname, 'groups_audio')),
+};
+
+const SHARD = String(args['shard-size-kb'] || 250);
+const CLEAN = Boolean(args['clean'] && String(args['clean']).toLowerCase() !== 'false');
+
+function runGroupManager(kind, sourceRoot, outDir) {
+  const script = path.join(__dirname, 'groupmanager.cjs');
+  const robotsDir = path.join(sourceRoot, 'robots');
+  const testsDir = path.join(sourceRoot, 'tests');
+  const publicDir = path.join(sourceRoot, 'public');
+  const snapshotsDir = path.join(sourceRoot, 'snapshots');
+  const args = [
+    script,
+    '--source-root', sourceRoot,
+    '--groups-dir', outDir,
+    '--robots-dir', robotsDir,
+    '--tests-dir', testsDir,
+    '--public-dir', publicDir,
+    '--snapshots-dir', snapshotsDir,
+    '--shard-size-kb', SHARD,
+  ];
+  if (CLEAN) args.push('--clean', 'true');
+  console.log(`[MAIN-MANAGER] 生成 ${kind} 大组: node ${args.join(' ')}`);
+  return new Promise((resolve, reject) => {
+    const p = spawn('node', args, { stdio: 'inherit' });
+    p.on('close', (code) => {
+      if (code === 0) resolve(); else reject(new Error(`${kind} 分组失败 code=${code}`));
+    });
+  });
+}
+
+async function main() {
+  // 确保输出根目录存在
+  for (const k of Object.keys(OUTPUTS)) {
+    fs.mkdirSync(OUTPUTS[k], { recursive: true });
+  }
+  // 依次生成三大组
+  await runGroupManager('text', SOURCES.text, OUTPUTS.text);
+  await runGroupManager('image', SOURCES.image, OUTPUTS.image);
+  await runGroupManager('audio', SOURCES.audio, OUTPUTS.audio);
+  console.log('[MAIN-MANAGER] 三个大组已准备完成:', OUTPUTS);
+  console.log('[MAIN-MANAGER] 后续可用 mainStarter.cjs 启动各自 GroupStarter');
+}
+
+main().catch((e) => {
+  console.error('[MAIN-MANAGER] 失败:', e.message);
+  process.exit(1);
+});