ezii 0.0.0.1

data/ezii-client/managables/services/livestream-interactive/chat.txt

@@ -0,0 +1,40 @@
+Welcome, GLHF!
+Your host is tmi.twitch.tv
+This server is rather new
+-
+-
+You are in a maze of twisty passages, all alike.
+>
+
+=
+#rw_grim
+hey
+Now Playing: Eye of Terror by Wontolla -> https://prtzl.to/kbHxL6DBFy
+wit.ai chat bot in training + me (manuel k.) as a human supervisor
+raffle?
+nice to meet u
+raffle
+The winner of this raffle is sillybaboy! [Follower]
+The winner of this raffle is morriscodes! [Follower]
+The winner of this raffle is makerblaker! [Follower]
+Follower
+:)
+!raffle
+madlads
+watsapp
+VoHiYo
+next wit.ai model training in a minute
+you say echo and you print it in the script
+Now Playing: Nightmare Court - edit by Wontolla -> https://prtzl.to/BJH9aYmAUN
+put it in single quotes
+add cool new features
+1:
+2:
+3:
+4:
+PLEASE
+Wait? is ezii_tm_registerred's owner here?
+yes
+
+do they have permission? @rw_grim
+PLEASE

data/ezii-client/managables/services/stdlib-dot-com/koma/qanda-api@dev/README.md

@@ -0,0 +1,3 @@
+# qanda-app.com API endpoints.
+
+A collection of API endpoints to find a random wikipedia sentence and black out a random word in it.

data/ezii-client/managables/services/stdlib-dot-com/koma/qanda-api@dev/functions/__main__.js

@@ -0,0 +1,58 @@
+const lib = require('lib');
+const _ = require('underscore');
+// const countWord = require('../statistics/count-word.js');
+
+/**
+* Qanda question endpoint.
+* @returns {object}
+*/
+module.exports = async (language = "en", source="wikipedia", context) => {
+  let randomSentenceResponse = await randomSentenceErrorProne(language, source, context);
+
+  let blackedOutDict = await lib[`${context.service.identifier}.black-out-random-word`]({sentence: randomSentenceResponse.rs.result});
+  let randomWordsFromArticle = await lib[`${context.service.identifier}.random-words`]({
+    similarTo: blackedOutDict.termNormal
+  });
+
+  let choices = [];
+
+  _.each(randomWordsFromArticle, (randomWord) => {
+    choices.push({correctAnswer: false, word: randomWord.word});
+  });
+
+  choices.push({correctAnswer: true, word: blackedOutDict.blackedOutWord});
+
+  choices = _.shuffle(choices);
+
+  let response = {
+    articleTitle: randomSentenceResponse.p.title, 
+    wikipediaId: randomSentenceResponse.p.wikipediaId, 
+    sentence: blackedOutDict, 
+    choices: choices,
+    surroundingSentences: randomSentenceResponse.rs.surroundingSentences
+  };
+
+  return response;
+}
+
+async function randomSentenceErrorProne(language, source, context) {
+  var text;
+  var page;
+  if (source === 'wikipedia') {
+    page = await lib[`${context.service.identifier}.random-wikipedia-page`]({language: language});
+    text = page.text;
+  } else {
+    page = {title: "Random texts from scan functionality (earliest alpha lel)"}
+    const ocrQuery = lib.koma['ocr-query']['@dev'];
+
+    text = await ocrQuery['random-ocr-text']();
+  }
+  
+  let randomSentence = await lib[`${context.service.identifier}.random-sentence`]({text: text});
+
+  if(randomSentence.error === true) {
+    return randomSentenceErrorProne(language, source, context);
+  } else {
+    return {p: page, rs: randomSentence};
+  }
+}

data/ezii-client/managables/services/stdlib-dot-com/koma/qanda-api@dev/functions/black-out-random-word.js

@@ -0,0 +1,74 @@
+const _ = require('underscore');
+const nlp = require('compromise');
+const lib = require('lib');
+
+/**
+* Black out a random word in a sentence. 
+* @returns {object}
+*/
+module.exports = async (sentence = "", context) => {
+  let sentenceTerms = nlp(sentence).out('terms');
+
+  let sentenceDict = {
+    before: "",
+    blackedOutWord: undefined,
+    after: ""
+  };
+  
+  sentenceTerms = _.filter(sentenceTerms, (term) => {
+    return term.normal !== "";
+  });
+  
+
+  var nounIndices = [];
+  for(i = 0; i < sentenceTerms.length; i++) {
+	    let term = sentenceTerms[i];
+            let isNoun = _.includes(term.tags, 'Noun');
+            let isLongerThan2Chars = term.normal.length > 2;
+
+	    if(isNoun && isLongerThan2Chars) {
+		    nounIndices.push(i);
+	    }
+  }
+
+  console.log(nounIndices);
+
+
+  let blackedOutIndex = _.sample(nounIndices);
+
+  console.log(blackedOutIndex);
+
+  _.each(sentenceTerms, (term, index)=> {   
+    if(index < blackedOutIndex) {
+      sentenceDict.before += term.text;
+      
+      if(index !== (blackedOutIndex - 1)) {
+        sentenceDict.before += " ";
+      }
+    } else if(index > blackedOutIndex) {
+      sentenceDict.after += term.text;
+      
+      if((sentenceTerms.length - 1) !== index) {
+        sentenceDict.after += " ";
+      }
+    } else if(index === blackedOutIndex) {
+      var termText = term.text;
+      
+      let specialCharsCharacterClass = '[,\\.\\?!()\\-\\–\/"\\s;:*]';
+      let nonWordCharsRegex = new RegExp(`^(${specialCharsCharacterClass}*)(.+?)(${specialCharsCharacterClass}*)$`);
+      let matchResult = nonWordCharsRegex.exec(termText);
+      
+      sentenceDict.termNormal = term.normal;
+      sentenceDict.tags = term.tags;
+      sentenceDict.before += matchResult[1];
+      sentenceDict.blackedOutWord = matchResult[2];
+      sentenceDict.after += matchResult[3];
+    }
+  });
+  
+  return sentenceDict;
+};
+
+function getRandomInt(min, max) {
+    return Math.floor(Math.random() * (max - min + 1)) + min;
+}

data/ezii-client/managables/services/stdlib-dot-com/koma/qanda-api@dev/functions/random-sentence.js

@@ -0,0 +1,57 @@
+const nlp = require('compromise');
+const _ = require('underscore');
+const lib = require('lib');
+
+/**
+* Get a random sentence from provided text. 
+* @returns {object}
+*/
+module.exports = async (text = "", wikiPediaPageId=0, context) => {
+  if(wikiPediaPageId !== 0) {
+    page = await lib[`${context.service.identifier}.random-wikipedia-page`]({pageIds: [wikiPediaPageId]});
+    text = page.text;
+  }
+
+  var sentencesDoc = nlp(text, {allowedTags: []}).sentences();
+  
+  sentences = sentencesDoc.list.map(ts => {
+    return ts.terms.map(t =>{
+      return t.text
+    });
+  });
+  
+  if(sentences.length === 0) {
+    return {error: true, message: 'No sentences found'};
+  }
+
+
+  function calculateRandomIndex(items) {
+    return Math.floor(Math.random()*items.length)
+  }
+
+  let randomIndex = calculateRandomIndex(sentences);
+  let sentence = sentences[randomIndex]
+  let tries = 0;
+  while(sentence.length < 5 && tries < 5) {
+    randomIndex = calculateRandomIndex(sentences);
+    sentence = sentences[randomIndex]
+    tries++;
+  }
+  
+  if(sentence.length < 5) {
+    return {error: true, message: 'No sentence with more than 5 words found'};
+  }
+
+  let sentencesBefore = [sentences[randomIndex-1].join(' ')];
+  let sentencesAfter = [sentences[randomIndex+1].join(' ')];
+
+  joinedSentence = sentence.join(' ');
+   return {
+    error: false,
+    result: joinedSentence,
+    surroundingSentences: {
+      before: sentencesBefore,
+      after: sentencesAfter
+    }
+  };
+};

data/ezii-client/managables/services/stdlib-dot-com/koma/qanda-api@dev/functions/random-wikipedia-page.js

@@ -0,0 +1,25 @@
+const axios = require('axios');
+const _ = require('underscore');
+const sanitizeHtml = require('sanitize-html');
+const decodeHtml = require('decode-html');
+
+/**
+* Random wikipedia article contents
+* @returns {object}
+*/
+module.exports = async (language = 'en') => {  
+  var apiResponse = await axios.get(randomWikipediaArticleApiEndpoint(language));
+  var page = _.values(apiResponse.data['query']['pages'])[0];
+  var response = {};
+  
+  response['text'] = decodeHtml(sanitizeHtml(page['extract'], {allowedTags: []}));
+  response['title'] = page['title'];
+  response['wikipediaId'] = page['pageid'];
+
+  return response;
+};
+
+function randomWikipediaArticleApiEndpoint(language) {
+  return `https://${language}.wikipedia.org/w/api.php?format=json&action=query&generator=random&grnnamespace=0&prop=extracts&grnlimit=1`;
+  //return `https://en.wikipedia.org/w/api.php?format=json&action=query&titles=World_history&grnnamespace=0&prop=extracts&grnlimit=1`;
+}

data/ezii-client/managables/services/stdlib-dot-com/koma/qanda-api@dev/functions/random-words.js

@@ -0,0 +1,34 @@
+const _ = require('underscore');
+const nlp = require('compromise');
+const w2v = require("word2vec-pure-js")
+w2v.load("./word2vec-models/test-text8-vector.bin")
+
+/**
+* Get random words form the provided text.
+* @returns {array}
+*/
+module.exports = async (numberOfWords = 3, similarTo = "") => {
+  let relatedTerms = _.shuffle(w2v.getSimilarWords(similarTo, numberOfWords ** 2));  
+
+  let terms = nlp(relatedTerms.join(" ")).out("terms");
+
+  terms = _.uniq(terms, 'normal');
+
+  terms = _.filter(terms, function(term) {
+	  return _.contains(term.tags, 'Noun');
+  });
+  
+  let relatedTermsEnrichedByCompromiseNlp = [];
+  for(i=0;i<numberOfWords;i++) {
+      let randIndex = Math.floor(Math.random()*terms.length);
+      relatedTermsEnrichedByCompromiseNlp.push(terms[randIndex]);
+      terms.splice(randIndex, 1);
+  }  
+
+
+  let randomWords = _.map(relatedTermsEnrichedByCompromiseNlp, (term, index)=> {
+    return {word: term.text, termNormal: term.normal};
+  });
+
+  return randomWords;
+};

data/ezii-client/managables/services/stdlib-dot-com/koma/qanda-api@dev/functions/rare-enough-word.js

@@ -0,0 +1,41 @@
+const lib = require('lib');
+// const l2nadb = require('./log-to-natural-database');
+
+/**
+* Get random words form the provided text.
+* @returns {any}
+*/
+module.exports = async function(word = "") {
+  //TODO: make database work again, natural db neeeds hosting
+  
+  // return "test";
+
+//   let queryMin = {
+//     "resource": "WordFrequency",
+//     "minColumn": "count"
+//   };
+// 
+//   let lessUsedWordCount = parseInt(
+//     (await lib[`koma.natural-db[@dev].min`](queryMin))
+//     ['result']
+//     [0]
+//   );
+// 
+//   let queryWordCount = {
+//     resource: 'WordFrequency',
+//     filter: 'word_normal',
+//     filterValue: word
+//   };
+// 
+//   let wordUsedCountQueryResult = (
+//     (await lib[`koma.natural-db[@dev].show`](queryWordCount))
+//     ['result']
+//   );
+// 
+//   if (wordUsedCountQueryResult.length === 0) return true;
+// 
+//   let wordUsedCount = parseInt(wordUsedCountQueryResult[0][2]);
+// 
+//   return (wordUsedCount < (lessUsedWordCount + 10));
+return word.length > 5;
+}

data/ezii-client/managables/services/stdlib-dot-com/koma/qanda-api@dev/functions/wikipedia-page.js

@@ -0,0 +1,35 @@
+const axios = require('axios');
+const _ = require('underscore');
+const sanitizeHtml = require('sanitize-html');
+const decodeHtml = require('decode-html');
+
+/**
+* Random wikipedia article contents
+* @returns {object}
+*/
+module.exports = async (language = 'en', pageIds = []) => {
+  var apiResponse;
+  if(pageIds) {
+    apiResponse = await axios.get(randomWikipediaArticleApiEndpoint(language));
+  } else {
+    apiResponse = await axios.get(artByPageId(pageIds[0]));
+  }
+  
+  var page = _.values(apiResponse.data['query']['pages'])[0];
+  var response = {};
+  
+  response['text'] = decodeHtml(sanitizeHtml(page['extract'], {allowedTags: []}));
+  response['title'] = page['title'];
+  response['wikipediaId'] = page['pageid'];
+
+  return response;
+};
+
+function randomWikipediaArticleApiEndpoint(language) {
+  return `https://${language}.wikipedia.org/w/api.php?format=json&action=query&generator=random&grnnamespace=0&prop=extracts&grnlimit=1`;
+  //return `https://en.wikipedia.org/w/api.php?format=json&action=query&titles=World_history&grnnamespace=0&prop=extracts&grnlimit=1`;
+}
+
+function artByPageId(pageId) {
+  return `https://en.wikipedia.org/w/api.php?format=json&action=query&pageids=${pageId}&grnnamespace=0&prop=extracts&grnlimit=1`
+}

data/ezii-client/managables/services/stdlib-dot-com/koma/qanda-api@dev/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "koma/qanda-api",
+  "description": "qanda-app.com API",
+  "version": "0.0.17",
+  "author": "koma",
+  "stdlib": {
+    "name": "koma/qanda-api",
+    "publish": true,
+    "timeout": 10000
+  },
+  "dependencies": {
+    "axios": "^0.18.0",
+    "underscore": "^1.9.1",
+    "compromise": "^11.9.1",
+    "sanitize-html": "^1.18.2",
+    "decode-html": "^2.0.0",
+    "lib": "^3.0.2",
+    "word2vec-pure-js": "^0.0.2"
+  }
+}

data/ezii-client/managables/services/stdlib-dot-com/koma/qanda-api@dev/statistics/count-word.js

@@ -0,0 +1,12 @@
+const lib = require('lib');
+
+module.exports = async function(word) {
+  await lib[`koma.natural-db[@dev].create-or-increment`]({
+    resource: 'WordFrequency',
+    whereColumn: 'word_normal',
+    whereValue: word,
+    incrementColumn: 'count',
+    token: process.env.NATURAL_DB_TOKEN,
+    db: 19
+  }); 
+}

data/ezii-client/managables/services/stdlib-dot-com/koma/qanda-api@dev/utils/log-to-natural-database.js

@@ -0,0 +1,10 @@
+const lib = require('lib');
+
+module.exports = async function(textToLog) {
+  await lib[`koma.natural-db[@dev].add`]({
+    resource: 'Logs',
+    data: {logged_text: textToLog},
+    token: "eyJhbGciOiJIUzI1NiJ9.eyJwcm9qZWN0X2lkIjoyMiwiZXhwIjoxNTYwMzU0ODM3fQ.jghaYoFOiMDWLQNNBZAWXYLE40DPwtMd5AN8-JyFpCI",
+    db: 46
+  });
+}

data/ezii-client/managament_interface/gems.locked

@@ -0,0 +1,13 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ruby2d (0.9.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  ruby2d
+
+BUNDLED WITH
+   2.0.2

data/ezii-client/managament_interface/gems.rb

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gem 'ruby2d'