@3-/aiapi 0.1.10 → 0.1.12

package/factCheck.js

@@ -0,0 +1,177 @@
+#!/usr/bin/env -S node --trace-uncaught --expose-gc --unhandled-rejections=strict --experimental-wasm-modules
+var PREFIX, SHOW_KIND, addUrl, circle, realUrl, txtLi, urlLi;
+
+import retry from '@3-/retry';
+
+import utf8d from '@3-/utf8/utf8d.js';
+
+import utf8e from '@3-/utf8/utf8e.js';
+
+// @3-/read
+// @3-/write
+SHOW_KIND = ['严重失实'];
+
+PREFIX = `对下文问答对做事实核查,只输出与事实不符的观点。
+必须搜索到明确的反面事实才算失实,模棱两可算无法核实。
+输出文本必须是简体中文，格式如下:
+- id:
+  标题: 要简短,一两个词即可
+  观点:
+  事实: 须标注搜索到的事实网页
+  失实度: ${SHOW_KIND.join(' / ')} / 略有出入 / 基本一致 / 无法核实
+---\n`;
+
+realUrl = async(url) => {
+  var err, r, status;
+  if (url.startsWith('http')) {
+    try {
+      r = (await fetch(url, {
+        method: 'HEAD',
+        redirect: 'follow'
+      }));
+      ({status} = r);
+      if (status === 200) {
+        ({url} = r);
+      } else {
+        console.error(status + ' :' + url);
+      }
+    } catch (error) {
+      err = error;
+      console.error(err);
+    }
+  }
+  return url;
+};
+
+circle = (num) => {
+  switch (false) {
+    case !(num > 0 && num <= 10):
+      return String.fromCodePoint(0x245F + num);
+    default:
+      return `[${num}]`;
+  }
+};
+
+urlLi = (id_li, url_li) => {
+  var i, r;
+  r = [];
+  for (i of id_li) {
+    r.push('[' + circle(i + 1) + '](' + url_li[i] + ')');
+  }
+  return r.join('');
+};
+
+addUrl = (text, items) => {
+  var bin, chunks, end, entry, j, len, output, pos, push, sorted, start;
+  output = '';
+  push = (bin) => {
+    output += utf8d(bin);
+  };
+  pos = 0;
+  sorted = items.sort(function(a, b) {
+    return a.segment.startIndex - b.segment.startIndex;
+  });
+  bin = utf8e(text);
+  for (j = 0, len = sorted.length; j < len; j++) {
+    entry = sorted[j];
+    ({
+      segment: {
+        startIndex: start,
+        endIndex: end
+      },
+      groundingChunkIndices: chunks
+    } = entry);
+    if (start > pos) {
+      push(bin.slice(pos, start));
+    }
+    push(bin.slice(start, end));
+    output += chunks;
+    pos = end;
+  }
+  if (pos < bin.length) {
+    push(bin.slice(pos));
+  }
+  return output;
+};
+
+txtLi = (txt) => {
+  var i, id, msg, pos, r, ref, t, tag, trim;
+  r = [];
+  ref = txt.replaceAll('**', '').split('\n');
+  for (i of ref) {
+    if (i.startsWith('- id:')) {
+      if (t) {
+        r.push(t);
+      }
+      id = parseInt(i.slice(5).trim());
+      if (id) {
+        t = {id};
+      }
+    } else if (t) {
+      trim = i.trimStart();
+      if (trim.length === i) {
+        r.push(t);
+        t = 0;
+      } else {
+        pos = trim.indexOf(':');
+        if (pos > 0) {
+          tag = trim.slice(0, pos).replace('- ', '').trim();
+          msg = trim.slice(pos + 1).trim();
+          t[tag] = msg;
+        }
+      }
+    }
+  }
+  if (t) {
+    r.push(t);
+  }
+  return r;
+};
+
+export default retry(async(chat, li) => {
+  var content, groundingChunks, groundingSupports, i, map, msg, out, ref, ref1, uri, url_li, x;
+  msg = PREFIX + li.map((i, pos) => {
+    return 'id:' + (pos + 1) + '\n' + i;
+  }).join('\n---\n');
+  out = (await chat(msg, 0, 0, {
+    tools: {
+      google_search: {}
+    }
+  }));
+  ({
+    // write '/tmp/fact.json',JSON.stringify out
+    // out = JSON.parse read '/tmp/fact.json'
+    content,
+    groundingMetadata: {groundingChunks, groundingSupports}
+  } = out);
+  if (!groundingChunks) {
+    return new Map();
+  }
+  content = content.parts[0].text;
+  url_li = [];
+  for (x of groundingChunks) {
+    ({
+      web: {uri}
+    } = x);
+    url_li.push(realUrl(uri));
+  }
+  url_li = (await Promise.all(url_li));
+  for (i of groundingSupports) {
+    i.groundingChunkIndices = urlLi(i.groundingChunkIndices, url_li);
+  }
+  map = new Map();
+  ref = txtLi(addUrl(content, groundingSupports));
+  for (i of ref) {
+    console.log(i);
+    if (!SHOW_KIND.includes((ref1 = i.失实度) != null ? ref1.split('[')[0] : void 0)) {
+      continue;
+    }
+    li = map.get(i.id);
+    if (!li) {
+      li = [];
+      map.set(i.id, li);
+    }
+    li.push(i);
+  }
+  return map;
+});

package/fmtSeg.js

@@ -5,6 +5,8 @@ import retry from '@3-/retry';
 
 export default retry(async(chat, txt) => {
   return (await chat(`将以下语音对话转为JSON数组,要确保所有问答都被包含,不要遗漏:\n` + txt, TOPIC_SCHEMA, "你是专业资深的秘书", {
-    temperature: 0
+    generationConfig: {
+      temperature: 0
+    }
   }));
 });

package/fmtWithFactCheck.js

@@ -0,0 +1,55 @@
+#!/usr/bin/env -S node --trace-uncaught --expose-gc --unhandled-rejections=strict --experimental-wasm-modules
+var check;
+
+import fmtJson from './fmtJson.js';
+
+import fmtJsonMd from './fmtJsonMd.js';
+
+import factCheck from './factCheck.js';
+
+import txtFmt from '@3-/txt_li/txtFmt.js';
+
+// @3-/read
+// @3-/write
+check = async(chat, json) => {
+  var j, k, li, n, x, y, z, 答, 问, 问答;
+  li = [];
+  for (x of json) {
+    ({问答} = x);
+    for (y of 问答) {
+      ({问, 答} = y);
+      li.push(`问:${txtFmt(问)}\n答:${txtFmt(答)}`);
+    }
+  }
+  check = (await factCheck(chat, li));
+  n = 0;
+  for (z of json) {
+    ({问答} = z);
+    for (j of 问答) {
+      li = check.get(++n);
+      if (li) {
+        for (k of li) {
+          j.答 += `\n#### ${k.失实度} : ${k.标题}
+
+> 观点: ${k.观点}
+>
+> 事实:
+>
+> ${k.事实}`;
+        }
+      }
+    }
+  }
+};
+
+export default async(chat, txt) => {
+  var json_li;
+  json_li = (await fmtJson(chat, txt));
+  // write('/tmp/chat.json', JSON.stringify(json_li))
+  // json_li = JSON.parse read('/tmp/chat.json')
+  // await check(chat, json_li[0])
+  await Promise.all(json_li.map((i) => {
+    return check(chat, i);
+  }));
+  return fmtJsonMd(json_li);
+};

package/gemini.js

@@ -1,6 +1,8 @@
 #!/usr/bin/env -S node --trace-uncaught --expose-gc --unhandled-rejections=strict --experimental-wasm-modules
 import sleep from '@3-/sleep';
 
+import merge from 'lodash-es/merge.js';
+
 export default (token_li) => {
   var _NEXT_TOKEN, _nextToken, nextToken;
   token_li.sort(() => {
@@ -18,21 +20,23 @@ export default (token_li) => {
   nextToken = () => {
     return _NEXT_TOKEN.next().value;
   };
-  return async(text, schema, system, generationConfig) => {
+  return async(text, schema, system, option) => {
     var body, err, error, message, r, status;
     body = {
-      generationConfig: {
-        responseMimeType: 'application/json',
-        responseJsonSchema: schema
-      },
       contents: [
         {
           parts: [{text}]
         }
       ]
     };
-    if (generationConfig) {
-      Object.assign(body.generationConfig, generationConfig);
+    if (schema) {
+      body.generationConfig = {
+        responseMimeType: 'application/json',
+        responseJsonSchema: schema
+      };
+    }
+    if (option) {
+      merge(body, option);
     }
     if (system) {
       body.system_instruction = {
@@ -87,13 +91,16 @@ export default (token_li) => {
         }
         throw new Error(text);
       }
-      r = ((await r.json())).candidates[0].content.parts[0].text;
-      try {
-        r = JSON.parse(r);
-      } catch (error1) {
-        err = error1;
-        console.error(r);
-        throw err;
+      r = ((await r.json())).candidates[0];
+      if (schema) {
+        r = r.content.parts[0].text;
+        try {
+          r = JSON.parse(r);
+        } catch (error1) {
+          err = error1;
+          console.error(r);
+          throw err;
+        }
       }
       return r;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@3-/aiapi",
-  "version": "0.1.10",
+  "version": "0.1.12",
   "repository": {
     "type": "git",
     "url": "git+https://atomgit.com/i18n/lib.git"
@@ -22,7 +22,9 @@
   "dependencies": {
     "@3-/retry": "^0.0.2",
     "@3-/sleep": "^0.0.4",
-    "@3-/txt_li": "^0.1.4"
+    "@3-/txt_li": "^0.1.4",
+    "@3-/utf8": "^0.0.4",
+    "lodash-es": "^4.17.21"
   },
   "scripts": {}
 }