casdoor 1.6.0 → 1.7.0

package/README.md

@@ -14,6 +14,12 @@ The Casdoor browser extension signs users into the systems that have no single s
 
 Casdoor only hands an account out for a page of the system's own sign-in URLs, records every time it does, and can keep the passwords encrypted (the organization's "Encrypted fields", with `fieldEncryptionKey` set). Form autofill needs a Casdoor Enterprise edition license. On a Casdoor without it, the extension fills the managed accounts on the pages that match the application's "Signin URL", a regular expression, as its first versions did.
 
+### Image captchas
+
+When **Recognize image captchas** is on in the application, the extension also reads the picture of characters on the sign-in page and types it in. It reads it with the [ddddocr](https://github.com/sml2h3/ddddocr) model in [ONNX Runtime Web](https://github.com/microsoft/onnxruntime), in a hidden page of the extension on the user's own computer: the picture is never sent anywhere, and it works on an intranet without internet access. The captcha input and picture are found automatically, or set with CSS selectors; the characters are digits when the page says so (e.g. "请输入数字验证码"), else letters and digits, or as the admin sets.
+
+When the sign-in page says the captcha was wrong, the extension tries a new one (it clicks the picture when the page doesn't change it), up to 3 times in all. When it says the username or password is wrong, it stops at once, so that the account doesn't get locked. A picture from another site can't be read by the extension, and then the user types the captcha. Sliders, picking pictures, SMS codes and third-party human checks (reCAPTCHA, GeeTest and the like) are not supported. Use it only on your own organization's systems.
+
 ## Usage
 
 ### Install the extension

package/manifest.json

@@ -1,16 +1,20 @@
 {
   "name": "Casdoor",
   "description": "Signs you into the systems without single sign-on by filling their sign-in forms with your accounts kept in Casdoor.",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "manifest_version": 3,
   "permissions": [
     "storage",
-    "identity"
+    "identity",
+    "offscreen"
   ],
   "host_permissions": [
     "https://*/*",
     "http://*/*"
   ],
+  "content_security_policy": {
+    "extension_pages": "script-src 'self' 'wasm-unsafe-eval'; object-src 'self'"
+  },
   "action": {
     "default_popup": "/src/popup.html",
     "default_icon": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "casdoor",
-  "version": "1.6.0",
+  "version": "1.7.0",
   "description": "The Casdoor extension can help browser users automatically log into applications protected by Casdoor single-sign-on (SSO) system.",
   "repository": {
     "type": "git",

package/src/background.js

@@ -156,6 +156,10 @@ async function getFill(sender) {
       usernameSelector: site.usernameSelector || "",
       passwordSelector: site.passwordSelector || "",
       submitSelector: site.submitSelector || "",
+      captcha: !!site.captcha,
+      captchaSelector: site.captchaSelector || "",
+      captchaImageSelector: site.captchaImageSelector || "",
+      captchaCharset: site.captchaCharset || "",
       autoSubmit: !!site.autoSubmit && !recentlySubmitted,
       urls: site.urls || [],
     },
@@ -181,6 +185,10 @@ async function getAccount(message, sender) {
   }
 
   const query = new URLSearchParams({application: site.application, username: message.username || "", url: sender.url});
+  // so that the audit record says the captcha of this sign-in was read by the extension
+  if (site.captcha) {
+    query.set("captcha", "true");
+  }
   const res = await callApi(`/api/get-form-fill-account?${query.toString()}`);
   if (res.status !== "ok") {
     return {error: res.msg};
@@ -191,6 +199,41 @@ async function getAccount(message, sender) {
   return {username: res.data.username, password: res.data.password};
 }
 
+const OFFSCREEN_URL = "/src/offscreen.html";
+let offscreenCreating = null;
+
+async function ensureOffscreen() {
+  const contexts = await chrome.runtime.getContexts({contextTypes: ["OFFSCREEN_DOCUMENT"], documentUrls: [chrome.runtime.getURL(OFFSCREEN_URL)]});
+  if (contexts.length > 0) {
+    return;
+  }
+  if (!offscreenCreating) {
+    offscreenCreating = chrome.offscreen.createDocument({
+      url: OFFSCREEN_URL,
+      reasons: ["WORKERS"],
+      justification: "Reads the image captcha of a sign-in page with a local model",
+    }).finally(() => {
+      offscreenCreating = null;
+    });
+  }
+  await offscreenCreating;
+}
+
+// recognizeCaptcha reads the captcha image of a sign-in page that the admin turned captcha recognition on for
+async function recognizeCaptcha(message, sender) {
+  const cache = await getSites();
+  const site = cache && findSite(cache, sender.url);
+  if (!site || !site.captcha || site.application !== message.application) {
+    return {error: "captcha recognition is off for this page"};
+  }
+  if (typeof message.image !== "string" || !message.image.startsWith("data:image/") || message.image.length > 2 * 1024 * 1024) {
+    return {error: "not a captcha image"};
+  }
+  await ensureOffscreen();
+  const res = await chrome.runtime.sendMessage({target: "offscreen", type: "recognizeCaptcha", image: message.image, charset: message.charset || ""});
+  return res || {error: "captcha recognition failed"};
+}
+
 function pendingKey(tabId) {
   return `pending-${tabId}`;
 }
@@ -333,6 +376,7 @@ async function getStatus() {
 const handlers = {
   getFill: (message, sender) => getFill(sender),
   getAccount: (message, sender) => getAccount(message, sender),
+  recognizeCaptcha: (message, sender) => recognizeCaptcha(message, sender),
   capture: (message, sender) => capture(message, sender),
   getPending: (message, sender) => getPending(sender),
   resolvePending: (message, sender) => resolvePending(message, sender),

package/src/content-script.js

@@ -33,6 +33,19 @@
       skip: "不用了",
       saved: "已保存到 Casdoor",
       close: "关闭",
+      missingSelector: "管理员配置的{field}选择器“{selector}”在这个页面上没有找到，已改用自动识别",
+      usernameField: "用户名输入框",
+      passwordField: "密码输入框",
+      submitField: "登录按钮",
+      captchaField: "验证码输入框",
+      captchaImageField: "验证码图片",
+      captchaNoInput: "没有找到验证码输入框",
+      captchaNoImage: "没有找到验证码图片",
+      captchaCrossOrigin: "验证码图片来自其他网站，插件读不到",
+      captchaUnread: "没有认出验证码",
+      captchaManual: "验证码请手动输入（{error}）",
+      captchaGaveUp: "验证码自动识别了 {n} 次都没有通过，请手动输入后登录",
+      signInRefused: "登录没有成功，可能是账号或密码不对，请检查托管账户",
     }
     : {
       filled: "Casdoor filled in the account {user} of {app}",
@@ -44,6 +57,19 @@
       skip: "No thanks",
       saved: "Saved to Casdoor",
       close: "Close",
+      missingSelector: "The {field} selector \"{selector}\" set by the admin matches nothing on this page, so it was found automatically",
+      usernameField: "username input",
+      passwordField: "password input",
+      submitField: "sign-in button",
+      captchaField: "captcha input",
+      captchaImageField: "captcha image",
+      captchaNoInput: "no captcha input was found",
+      captchaNoImage: "no captcha image was found",
+      captchaCrossOrigin: "the captcha image comes from another site, the extension can't read it",
+      captchaUnread: "the captcha could not be read",
+      captchaManual: "Please type the captcha ({error})",
+      captchaGaveUp: "The captcha was read {n} times without success, please type it and sign in",
+      signInRefused: "The sign-in failed, the username or the password may be wrong. Please check the managed account",
     };
 
   function format(text, values) {
@@ -78,19 +104,36 @@
     return element.getClientRects().length > 0;
   }
 
+  // a class attribute pasted as it is, like "el-button login-btn el-button--primary", is taken as those classes
+  function selectorCandidates(selector) {
+    const value = selector.trim();
+    const candidates = [value];
+    if (/^[\w-]+(\s+[\w-]+)*$/.test(value)) {
+      candidates.push("." + value.split(/\s+/).join("."));
+    }
+    return candidates;
+  }
+
   function query(selector) {
-    if (!selector) {
+    if (!selector || !selector.trim()) {
       return null;
     }
-    try {
-      return document.querySelector(selector);
-    } catch (error) {
-      return null;
+    for (const candidate of selectorCandidates(selector)) {
+      try {
+        const element = document.querySelector(candidate);
+        if (element) {
+          return element;
+        }
+      } catch (error) {
+        // not a valid selector, try the next way of reading it
+      }
     }
+    return null;
   }
 
   const usernameHints = /user|login|account|email|mail|phone|mobile|tel|name|uid|uname|loginid|工号|账号|帳號|用户|用戶|手机|邮箱/i;
-  const submitHints = /^\s*(登\s*录|登\s*入|登\s*陆|log\s*in|sign\s*in|login|signin|submit|确\s*定|提\s*交|进入系统)\s*$/i;
+  // "登录", "登录系统", "立即登录", "Sign in", ...
+  const submitHints = /^\s*(?:(?:立即|马上|安全|用户|账号)?\s*(?:登\s*录|登\s*入|登\s*陆)\s*(?:系统|平台)?|进入系统|log\s*in|log\s*on|sign\s*in|login|logon|signin|submit|确\s*定|提\s*交)\s*$/i;
 
   function findPassword(site) {
     const configured = query(site.passwordSelector);
@@ -136,8 +179,14 @@
         return button;
       }
     }
-    const clickable = [...document.querySelectorAll("button, input[type=button], input[type=submit], a, [role=button], div[onclick], span[onclick]")];
-    return clickable.find((element) => isVisible(element) && submitHints.test(element.value || element.textContent || "")) || null;
+    // buttons first, so that a link or a tab with the same words loses to the real button
+    for (const selector of ["button, input[type=button], input[type=submit]", "[role=button]", "a, div[onclick], span[onclick]"]) {
+      const found = [...document.querySelectorAll(selector)].find((element) => isVisible(element) && submitHints.test(element.value || element.textContent || ""));
+      if (found) {
+        return found;
+      }
+    }
+    return null;
   }
 
   function setValue(input, value) {
@@ -154,8 +203,11 @@
     for (const type of ["keydown", "keypress", "keyup"]) {
       input.dispatchEvent(new KeyboardEvent(type, {key: "Enter", code: "Enter", keyCode: 13, which: 13, bubbles: true}));
     }
-    if (input.form) {
-      input.form.requestSubmit();
+    // a single-page app (Vue, React, ...) signs in from its own Enter handler, submitting its <form> would only reload
+    // the page; a server-side form says where it goes
+    const form = input.form;
+    if (form && (form.hasAttribute("action") || form.hasAttribute("method"))) {
+      form.requestSubmit();
     }
   }
 
@@ -187,6 +239,7 @@
     title.style.cssText = "font-weight:600;font-size:12px;color:#5c6ac4;margin-bottom:4px;";
     title.textContent = "Casdoor";
     const body = document.createElement("div");
+    body.style.whiteSpace = "pre-line";
     body.textContent = text;
     box.append(title, body);
     if (buttons.length > 0) {
@@ -215,6 +268,199 @@
     }
   }
 
+  // captchas
+
+  const captchaInputHints = /验证码|校验码|captcha|verif|vcode|checkcode|check_code|yzm|randcode|validcode|valicode|imgcode|authcode|securitycode|kaptcha/i;
+  const captchaImageHints = /验证码|captcha|verif|vcode|checkcode|kaptcha|yzm|randcode|validcode|valicode|imgcode|authcode|code/i;
+  const captchaErrorHints = /(验证码|校验码|captcha|verification code|security code).{0,12}(错|不正确|不对|有误|无效|失效|过期|incorrect|invalid|wrong|expired)|(错误|不正确|无效|incorrect|invalid|wrong).{0,6}(验证码|captcha)/i;
+  const passwordErrorHints = /(密码|口令|账号|用户名|用户|password|username|account).{0,12}(错|不正确|不对|有误|不存在|锁定|incorrect|invalid|wrong|locked)/i;
+  const MAX_CAPTCHA_TRIES = 3;
+
+  function isShown(element) {
+    if (!element || !element.isConnected) {
+      return false;
+    }
+    const style = window.getComputedStyle(element);
+    return style.visibility !== "hidden" && style.display !== "none" && element.getClientRects().length > 0;
+  }
+
+  function findCaptchaInput(site, usernameInput, password) {
+    const configured = query(site.captchaSelector);
+    if (configured) {
+      return configured;
+    }
+    const inputs = [...document.querySelectorAll("input")].filter((input) => isTextInput(input) && isVisible(input) && input !== usernameInput && input !== password);
+    return inputs.find((input) => captchaInputHints.test(`${input.name} ${input.id} ${input.placeholder} ${input.className} ${input.getAttribute("aria-label") || ""} ${input.getAttribute("autocomplete") || ""}`) ||
+      /(^|[^a-z])code([^a-z]|$)/i.test(`${input.name} ${input.id}`)) || null;
+  }
+
+  function center(element) {
+    const rect = element.getBoundingClientRect();
+    return [rect.left + rect.width / 2, rect.top + rect.height / 2];
+  }
+
+  // the captcha picture: the one the admin set, or an image or canvas of a captcha's size, the hinted and closest first
+  function findCaptchaImage(site, input) {
+    const configured = query(site.captchaImageSelector);
+    if (configured) {
+      return configured;
+    }
+    const [x, y] = center(input);
+    const candidates = [...document.querySelectorAll("img, canvas")].filter((element) => {
+      if (!isShown(element)) {
+        return false;
+      }
+      const rect = element.getBoundingClientRect();
+      return rect.width >= 40 && rect.width <= 320 && rect.height >= 15 && rect.height <= 120;
+    }).map((element) => {
+      const [cx, cy] = center(element);
+      const hinted = captchaImageHints.test(`${element.getAttribute("src") || ""} ${element.id} ${element.className} ${element.getAttribute("alt") || ""} ${element.getAttribute("title") || ""}`);
+      return {element, score: Math.hypot(cx - x, cy - y) - (hinted ? 10000 : 0)};
+    });
+    candidates.sort((a, b) => a.score - b.score);
+    return candidates.length > 0 ? candidates[0].element : null;
+  }
+
+  function waitForImage(image, timeout) {
+    if (!(image instanceof HTMLImageElement) || (image.complete && image.naturalWidth > 0)) {
+      return Promise.resolve();
+    }
+    return new Promise((resolve) => {
+      const done = () => resolve();
+      image.addEventListener("load", done, {once: true});
+      image.addEventListener("error", done, {once: true});
+      setTimeout(done, timeout);
+    });
+  }
+
+  // the picture as it is on the page; loading its address again would give a new captcha
+  async function readCaptchaImage(image) {
+    if (image instanceof HTMLCanvasElement) {
+      return image.toDataURL("image/png");
+    }
+    if (!(image instanceof HTMLImageElement)) {
+      throw new Error(strings.captchaNoImage);
+    }
+    await waitForImage(image, 3000);
+    if (!image.naturalWidth) {
+      throw new Error(strings.captchaNoImage);
+    }
+    const canvas = document.createElement("canvas");
+    canvas.width = image.naturalWidth;
+    canvas.height = image.naturalHeight;
+    canvas.getContext("2d").drawImage(image, 0, 0);
+    try {
+      return canvas.toDataURL("image/png");
+    } catch (error) {
+      // an image of another site without CORS taints the canvas
+      throw new Error(strings.captchaCrossOrigin);
+    }
+  }
+
+  function captchaCharset(site, input) {
+    if (site.captchaCharset) {
+      return site.captchaCharset;
+    }
+    const label = input.closest("label, .el-form-item, .ant-form-item, .form-group, tr, li");
+    const text = `${input.placeholder} ${input.getAttribute("aria-label") || ""} ${input.title} ${label ? label.textContent : ""}`;
+    return /数字|digit|number/i.test(text) ? "Digits" : "Alphanumeric";
+  }
+
+  async function solveCaptcha(site, input) {
+    const image = findCaptchaImage(site, input);
+    if (!image) {
+      throw new Error(strings.captchaNoImage);
+    }
+    const data = await readCaptchaImage(image);
+    const res = await send({type: "recognizeCaptcha", application: site.application, image: data, charset: captchaCharset(site, input)});
+    if (!res || res.error || !res.text) {
+      throw new Error((res && res.error) || strings.captchaUnread);
+    }
+    let text = res.text;
+    if (input.maxLength > 0 && text.length > input.maxLength) {
+      text = text.slice(0, input.maxLength);
+    }
+    setValue(input, text);
+    return {data};
+  }
+
+  // a new captcha: most sign-in pages change it when the picture is clicked, some already did on the failed sign-in
+  async function refreshCaptcha(site, input, before) {
+    const image = findCaptchaImage(site, input);
+    if (!image) {
+      return;
+    }
+    const current = await readCaptchaImage(image).catch(() => "");
+    if (current && current !== before.data) {
+      return;
+    }
+    image.click();
+    for (let i = 0; i < 15; i++) {
+      await sleep(200);
+      const latest = findCaptchaImage(site, input);
+      if (latest) {
+        await waitForImage(latest, 1000);
+        const data = await readCaptchaImage(latest).catch(() => "");
+        if (data && data !== before.data) {
+          return;
+        }
+      }
+    }
+  }
+
+  function submitForm(site, password) {
+    const submit = findSubmit(site, password);
+    if (submit) {
+      submit.click();
+    } else {
+      pressEnter(password);
+    }
+  }
+
+  // After submitting with a recognized captcha: still on the sign-in form means it was refused. A wrong captcha is
+  // tried again with a new one, a wrong password is not, so that the account doesn't get locked.
+  async function watchCaptchaSignIn(site, account, password, input, attempt, solved, textBefore) {
+    const url = location.href;
+    await sleep(2500);
+    const stillHere = location.href === url && (isShown(password) || isShown(findPassword(site)));
+    if (!stillHere) {
+      return;
+    }
+    const now = document.body.innerText;
+    const added = now.split(textBefore).join(" ");
+    const captchaWrong = captchaErrorHints.test(added) || captchaErrorHints.test(now);
+    const passwordWrong = !captchaWrong && passwordErrorHints.test(added);
+    if (passwordWrong || attempt >= MAX_CAPTCHA_TRIES || (!captchaWrong && attempt >= 2)) {
+      const currentInput = isShown(input) ? input : findCaptchaInput(site, null, findPassword(site));
+      if (currentInput) {
+        setValue(currentInput, "");
+        currentInput.focus();
+      }
+      showNotice(passwordWrong ? strings.signInRefused : format(strings.captchaGaveUp, {n: attempt}), [], 0);
+      return;
+    }
+
+    const currentPassword = isShown(password) ? password : findPassword(site);
+    const currentInput = isShown(input) ? input : findCaptchaInput(site, findUsername(site, currentPassword), currentPassword);
+    if (!currentPassword || !currentInput) {
+      return;
+    }
+    if (!currentPassword.value) {
+      setValue(currentPassword, account.password);
+    }
+    try {
+      await refreshCaptcha(site, currentInput, solved);
+      const next = await solveCaptcha(site, currentInput);
+      const text = document.body.innerText;
+      await sleep(300);
+      submitForm(site, currentPassword);
+      await watchCaptchaSignIn(site, account, currentPassword, currentInput, attempt + 1, next, text);
+    } catch (error) {
+      currentInput.focus();
+      showNotice(format(strings.captchaManual, {error: error.message}), [], 0);
+    }
+  }
+
   async function fill(info, username) {
     const site = info.site;
     // a page of the system without a sign-in form, e.g. when the URLs end with "*"
@@ -233,21 +479,56 @@
     }
     setValue(password, account.password);
 
+    // the captcha is read on the user's computer, see offscreen.js
+    let captchaInput = null;
+    let solved = null;
+    let captchaError = "";
+    if (site.captcha) {
+      captchaInput = findCaptchaInput(site, usernameInput, password);
+      if (!captchaInput) {
+        captchaError = strings.captchaNoInput;
+      } else {
+        try {
+          solved = await solveCaptcha(site, captchaInput);
+        } catch (error) {
+          captchaError = error.message;
+        }
+      }
+    }
+
     const others = info.accounts.filter((item) => item !== account.username);
     const buttons = others.slice(0, 3).map((item) => ({label: `${strings.switchAccount} ${item}`, onClick: () => fill(info, item)}));
     if (info.recentlySubmitted && info.accounts.length > 0) {
       showNotice(strings.retry, buttons, 0);
       return;
     }
-    showNotice(format(strings.filled, {app: site.displayName, user: account.username}), buttons);
+    // the admin's selectors that match nothing, so that a wrong one doesn't go unnoticed
+    const missing = [
+      [site.usernameSelector, strings.usernameField],
+      [site.passwordSelector, strings.passwordField],
+      [site.autoSubmit ? site.submitSelector : "", strings.submitField],
+      [site.captcha ? site.captchaSelector : "", strings.captchaField],
+      [site.captcha ? site.captchaImageSelector : "", strings.captchaImageField],
+    ].filter(([selector]) => selector && selector.trim() && !query(selector));
+    const warnings = missing.map(([selector, field]) => format(strings.missingSelector, {field, selector}));
+    if (captchaError) {
+      warnings.push(format(strings.captchaManual, {error: captchaError}));
+    }
+    showNotice([format(strings.filled, {app: site.displayName, user: account.username}), ...warnings].join("\n"), buttons, warnings.length > 0 ? 15000 : 8000);
 
+    if (captchaError) {
+      // nothing is submitted without the captcha, the user types it
+      if (captchaInput) {
+        captchaInput.focus();
+      }
+      return;
+    }
     if (site.autoSubmit && !username) {
       await sleep(400);
-      const submit = findSubmit(site, password);
-      if (submit) {
-        submit.click();
-      } else {
-        pressEnter(password);
+      const textBefore = document.body.innerText;
+      submitForm(site, password);
+      if (solved) {
+        watchCaptchaSignIn(site, account, password, captchaInput, 1, solved, textBefore);
       }
     }
   }

package/src/offscreen.html

@@ -0,0 +1,5 @@
+<!doctype html>
+<meta charset="utf-8">
+<title>Casdoor captcha recognition</title>
+<script src="/vendor/onnxruntime-web/ort.wasm.min.js"></script>
+<script src="/src/offscreen.js"></script>

package/src/offscreen.js

@@ -0,0 +1,125 @@
+// Copyright 2026 The Casdoor Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Reads the image captcha of a sign-in page with ddddocr's model in ONNX Runtime Web, on the user's own computer: the
+// image never leaves it. This hidden page exists because a service worker can't run WebAssembly with a canvas.
+
+/* global ort */
+
+const MODEL_HEIGHT = 64;
+
+let model = null;
+
+async function loadModel() {
+  if (!model) {
+    model = (async() => {
+      ort.env.wasm.wasmPaths = chrome.runtime.getURL("/vendor/onnxruntime-web/");
+      // threads need a cross-origin isolated page
+      ort.env.wasm.numThreads = 1;
+      const [session, charset] = await Promise.all([
+        ort.InferenceSession.create(chrome.runtime.getURL("/vendor/ddddocr/common_old.onnx"), {executionProviders: ["wasm"]}),
+        fetch(chrome.runtime.getURL("/vendor/ddddocr/charset.json")).then((res) => res.json()),
+      ]);
+      return {session, charset};
+    })();
+    model.catch(() => {
+      model = null;
+    });
+  }
+  return model;
+}
+
+function loadImage(dataUrl) {
+  return new Promise((resolve, reject) => {
+    const image = new Image();
+    image.onload = () => resolve(image);
+    image.onerror = () => reject(new Error("the captcha image could not be read"));
+    image.src = dataUrl;
+  });
+}
+
+// the same input as ddddocr: scaled to 64 pixels high, grey (ITU-R 601-2 luma, as PIL's "L"), from -1 to 1
+async function toTensor(dataUrl) {
+  const image = await loadImage(dataUrl);
+  const width = Math.max(1, Math.round(image.naturalWidth * MODEL_HEIGHT / image.naturalHeight));
+  const canvas = document.createElement("canvas");
+  canvas.width = width;
+  canvas.height = MODEL_HEIGHT;
+  const context = canvas.getContext("2d", {willReadFrequently: true});
+  // a transparent captcha is read on white
+  context.fillStyle = "#fff";
+  context.fillRect(0, 0, width, MODEL_HEIGHT);
+  context.imageSmoothingEnabled = true;
+  context.imageSmoothingQuality = "high";
+  context.drawImage(image, 0, 0, width, MODEL_HEIGHT);
+  const pixels = context.getImageData(0, 0, width, MODEL_HEIGHT).data;
+  const data = new Float32Array(width * MODEL_HEIGHT);
+  for (let i = 0; i < data.length; i++) {
+    const grey = Math.round((pixels[i * 4] * 299 + pixels[i * 4 + 1] * 587 + pixels[i * 4 + 2] * 114) / 1000);
+    data[i] = (grey / 255 - 0.5) / 0.5;
+  }
+  return new ort.Tensor("float32", data, [1, 1, MODEL_HEIGHT, width]);
+}
+
+const charsetPatterns = {
+  Digits: /^[0-9]$/,
+  Alphanumeric: /^[0-9a-zA-Z]$/,
+};
+
+// CTC decoding of the model's output [steps, 1, classes], only over the characters the captcha can have
+function decode(output, charset, allowedName) {
+  const [steps, , classes] = output.dims;
+  const pattern = charsetPatterns[allowedName];
+  const allowed = [];
+  for (let c = 0; c < classes; c++) {
+    if (c === 0 || !pattern || pattern.test(charset[c])) {
+      allowed.push(c);
+    }
+  }
+  let result = "";
+  let last = 0;
+  for (let t = 0; t < steps; t++) {
+    let best = 0;
+    let bestScore = -Infinity;
+    for (const c of allowed) {
+      const score = output.data[t * classes + c];
+      if (score > bestScore) {
+        bestScore = score;
+        best = c;
+      }
+    }
+    if (best !== last && best !== 0) {
+      result += charset[best];
+    }
+    last = best;
+  }
+  return result;
+}
+
+async function recognize(image, charsetName) {
+  const {session, charset} = await loadModel();
+  const input = await toTensor(image);
+  const outputs = await session.run({[session.inputNames[0]]: input});
+  return decode(outputs[session.outputNames[0]], charset, charsetName);
+}
+
+chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
+  if (!message || message.target !== "offscreen" || message.type !== "recognizeCaptcha") {
+    return false;
+  }
+  recognize(message.image, message.charset)
+    .then((text) => sendResponse({text}))
+    .catch((error) => sendResponse({error: error.message}));
+  return true;
+});

package/vendor/README.md

@@ -0,0 +1,6 @@
+# Third-party files
+
+- `onnxruntime-web/`: the WebAssembly build of [ONNX Runtime Web](https://github.com/microsoft/onnxruntime) 1.22.0 (`ort.wasm.min.js`, `ort-wasm-simd-threaded.mjs`, `ort-wasm-simd-threaded.wasm` from the npm package `onnxruntime-web@1.22.0`), MIT License.
+- `ddddocr/`: the captcha recognition model `common_old.onnx` and its character set (`charset.json`, taken from `ddddocr/__init__.py`) of [ddddocr](https://github.com/sml2h3/ddddocr) 1.5.6 (the PyPI wheel), MIT License.
+
+They are used by `src/offscreen.js` to read the image captchas of sign-in pages on the user's own computer.

package/vendor/ddddocr/LICENSE

@@ -0,0 +1,8 @@
+The MIT License (MIT)
+Copyright © 2022 <copyright holders>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.