079project 6.0.0 → 8.0.0

package/memeMergeWorker.cjs

@@ -0,0 +1,256 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const workerpool = require('workerpool');
+const csvParse = require('csv-parse/sync');
+const crypto = require('crypto');
+
+const safeRequire = (name) => {
+    try {
+        return require(name);
+    } catch (_err) {
+        return null;
+    }
+};
+
+const natural = safeRequire('natural');
+const porterStemmer = natural?.PorterStemmer ?? null;
+
+const lemmaCache = new Map();
+const lemmaMTime = new Map();
+
+const resolveLemmaPath = (lemmaCsvPath) => {
+    if (!lemmaCsvPath) {
+        return null;
+    }
+    if (path.isAbsolute(lemmaCsvPath)) {
+        return lemmaCsvPath;
+    }
+    return path.resolve(process.cwd(), lemmaCsvPath);
+};
+
+const loadLemmaMap = (lemmaCsvPath) => {
+    const resolvedPath = resolveLemmaPath(lemmaCsvPath);
+    if (!resolvedPath) {
+        return new Map();
+    }
+    try {
+        const stat = fs.statSync(resolvedPath);
+        const cached = lemmaCache.get(resolvedPath);
+        const cachedMTime = lemmaMTime.get(resolvedPath);
+        if (cached && cachedMTime && cachedMTime >= stat.mtimeMs) {
+            return cached;
+        }
+        const csvContent = fs.readFileSync(resolvedPath, 'utf8');
+        const records = csvParse.parse(csvContent, {
+            skip_empty_lines: true,
+            relax_column_count: true
+        });
+        const map = new Map();
+        for (const row of records) {
+            if (!Array.isArray(row)) {
+                continue;
+            }
+            const tokens = row
+                .map((item) => String(item || '').trim().toLowerCase())
+                .filter(Boolean);
+            if (!tokens.length) {
+                continue;
+            }
+            const base = tokens[0];
+            for (const token of tokens) {
+                map.set(token, base);
+            }
+        }
+        lemmaCache.set(resolvedPath, map);
+        lemmaMTime.set(resolvedPath, stat.mtimeMs);
+        return map;
+    } catch (_err) {
+        return new Map();
+    }
+};
+
+const normalizeWord = (word) => {
+    if (word == null) {
+        return '';
+    }
+    return String(word).trim().toLowerCase();
+};
+
+const lemmatizeToken = (word, lemmaMap) => {
+    const normalized = normalizeWord(word);
+    if (!normalized) {
+        return normalized;
+    }
+    const lemma = lemmaMap.get(normalized);
+    if (lemma) {
+        return lemma;
+    }
+    if (porterStemmer && porterStemmer.stem) {
+        return porterStemmer.stem(normalized);
+    }
+    return normalized;
+};
+
+const batchLemmatize = (sentences, lemmaCsvPath) => {
+    if (!Array.isArray(sentences)) {
+        return [];
+    }
+    const lemmaMap = loadLemmaMap(lemmaCsvPath);
+    return sentences.map((sentence) => {
+        if (!Array.isArray(sentence)) {
+            return [];
+        }
+        return sentence.map((word) => lemmatizeToken(word, lemmaMap));
+    });
+};
+
+const levenshtein = (a, b) => {
+    if (a === b) {
+        return 0;
+    }
+    const lenA = a.length;
+    const lenB = b.length;
+    if (lenA === 0) {
+        return lenB;
+    }
+    if (lenB === 0) {
+        return lenA;
+    }
+    const prev = new Array(lenB + 1);
+    const curr = new Array(lenB + 1);
+    for (let j = 0; j <= lenB; j++) {
+        prev[j] = j;
+    }
+    for (let i = 1; i <= lenA; i++) {
+        curr[0] = i;
+        const charA = a.charCodeAt(i - 1);
+        for (let j = 1; j <= lenB; j++) {
+            const charB = b.charCodeAt(j - 1);
+            const cost = charA === charB ? 0 : 1;
+            curr[j] = Math.min(
+                prev[j] + 1,
+                curr[j - 1] + 1,
+                prev[j - 1] + cost
+            );
+        }
+        for (let j = 0; j <= lenB; j++) {
+            prev[j] = curr[j];
+        }
+    }
+    return prev[lenB];
+};
+
+const uniqueWords = (words) => {
+    const seen = new Set();
+    const out = [];
+    for (const word of words) {
+        const normalized = normalizeWord(word);
+        if (!normalized || seen.has(normalized)) {
+            continue;
+        }
+        seen.add(normalized);
+        out.push(normalized);
+    }
+    return out;
+};
+
+const hashWord = (word) => {
+    return crypto.createHash('sha1').update(word).digest('hex');
+};
+
+const calcMemeDistance = ({ wordsA = [], wordsB = [] } = {}) => {
+    const normA = uniqueWords(wordsA);
+    const normB = uniqueWords(wordsB);
+    if (!normA.length && !normB.length) {
+        return { avgDist: 0, overlap: 0, jaccard: 0, memeHashes: [] };
+    }
+    const setA = new Set(normA);
+    const setB = new Set(normB);
+    let overlap = 0;
+    for (const word of setA) {
+        if (setB.has(word)) {
+            overlap += 1;
+        }
+    }
+    let totalDistance = 0;
+    if (!normA.length || !normB.length) {
+        totalDistance = normA.length + normB.length;
+    } else {
+        const accumulate = (source, target) => {
+            for (const word of source) {
+                let best = Infinity;
+                for (const candidate of target) {
+                    if (word === candidate) {
+                        best = 0;
+                        break;
+                    }
+                    const dist = levenshtein(word, candidate);
+                    const normalized = dist / Math.max(word.length, candidate.length, 1);
+                    if (normalized < best) {
+                        best = normalized;
+                    }
+                }
+                totalDistance += Number.isFinite(best) ? best : 1;
+            }
+        };
+        accumulate(normA, normB);
+        accumulate(normB, normA);
+    }
+    const avgDist = (normA.length + normB.length) > 0
+        ? totalDistance / (normA.length + normB.length)
+        : 0;
+    const unionSize = new Set([...setA, ...setB]).size;
+    const jaccard = unionSize > 0 ? overlap / unionSize : 0;
+    const memeHashes = [
+        normA.map((word) => hashWord(word)),
+        normB.map((word) => hashWord(word))
+    ];
+    return { avgDist, overlap, jaccard, memeHashes };
+};
+
+const clonePoints = (pointsArr) => {
+    if (pointsArr instanceof Map) {
+        return clonePoints(Array.from(pointsArr.entries()));
+    }
+    if (!Array.isArray(pointsArr)) {
+        return [];
+    }
+    return pointsArr.map((entry) => {
+        const [key, value] = Array.isArray(entry)
+            ? entry
+            : [entry?.key, entry?.value];
+        if (key === undefined) {
+            return [undefined, undefined];
+        }
+        const connect = Array.isArray(value?.connect)
+            ? value.connect.map((item) => {
+                if (Array.isArray(item)) {
+                    return [...item];
+                }
+                if (item && typeof item === 'object') {
+                    return { ...item };
+                }
+                return item;
+            })
+            : [];
+        const cloned = {
+            pointID: value?.pointID ?? null,
+            connect
+        };
+        if (value?.meta && typeof value.meta === 'object') {
+            cloned.meta = { ...value.meta };
+        }
+        if (value?.vector && Array.isArray(value.vector)) {
+            cloned.vector = value.vector.slice();
+        }
+        return [key, cloned];
+    });
+};
+
+workerpool.worker({
+    calcMemeDistance,
+    clonePoints,
+    batchLemmatize
+});

package/package.json

@@ -1,16 +1,14 @@
 {
   "name": "079project",
-  "version": "6.0.0",
-  "description": "a GNN-GA BASED ai that might pass the turing test,which use little resources.its startpoint initialize it and you can start it as ```node mainStarter.cjs```",
+  "version": "8.0.0",
+  "description": "a matrix based ai that can performance well in little source of words,which use little resources.its startpoint initialize it and you can start it as the description in readme.md",
   "keywords": [
     "ai",
-    "gat",
+    "matrix",
     "backend",
-    "gnn",
+    "transformer",
     "artificial",
-    "intelligence",
-    "turing",
-    "test"
+    "intelligence"
   ],
   "repository": {
     "type": "git",
@@ -19,12 +17,14 @@
   "license": "LGPL-3.0",
   "author": "mumu2009",
   "type": "commonjs",
-  "main": "mainManager.cjs",
+  "main": "main.cjs",
   "directories": {
     "test": "tests"
   },
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "start:ai": "node .\\main.cjs",
+    "start:auth": "node .\\auth_frontend_server.cjs"
   },
   "dependencies": {
     "@redis/bloom": "^5.8.3",
@@ -33,35 +33,48 @@
     "@redis/search": "^5.8.3",
     "@tensorflow/tfjs": "^4.22.0",
     "@tensorflow/tfjs-layers": "^4.22.0",
-    "axios": "^1.11.0",
-    "body-parser": "^2.2.0",
+    "axios": "^1.13.2",
+    "bcryptjs": "^3.0.3",
+    "better-sqlite3": "^12.5.0",
+    "body-parser": "^2.2.1",
     "brainjs": "^0.7.4",
     "chalk": "^4.1.2",
+    "cheerio": "^1.1.2",
     "child_process": "^1.0.2",
+    "csv-parse": "^6.1.0",
     "debug": "^4.4.3",
     "ee-first": "^1.1.1",
     "electron": "^37.2.4",
     "encodeurl": "^2.0.0",
     "escape-html": "^1.0.3",
-    "express": "^4.0.0",
+    "express": "^5.2.1",
     "follow-redirects": "^1.15.11",
     "forwarded": "^0.2.0",
     "ipaddr.js": "^2.3.0",
+    "jsonwebtoken": "^9.0.3",
     "lmdb": "^3.4.3",
+    "ml-kmeans": "^7.0.0",
+    "ml-matrix": "^6.12.1",
     "ms": "^2.1.3",
     "natural": "^8.1.0",
+    "numeric": "^1.2.6",
     "object-inspect": "^1.13.4",
     "on-finished": "^2.4.1",
     "parseurl": "^1.3.3",
+    "path-to-regexp": "^8.3.0",
+    "pdf-parse": "^2.4.5",
     "protobufjs": "^7.5.4",
     "puppeteer": "^24.26.1",
-    "redis": "^5.8.3",
+    "redis": "^5.10.0",
+    "seedrandom": "^3.0.5",
     "side-channel": "^1.1.0",
     "statuses": "^2.0.2",
     "synaptic": "^1.1.4",
     "tensorflowjs": "^0.6.8",
     "umap-js": "^1.4.0",
-    "workerpool": "^9.3.3"
+    "workerpool": "^10.0.1"
   },
-  "devDependencies": {}
+  "devDependencies": {
+    "parcel-bundler": "^1.12.5"
+  }
 }