anais-apk-forensic 1.0.0

package/apkid/apkid.py

@@ -0,0 +1,266 @@
+"""
+ Copyright (C) 2024  RedNaga. https://rednaga.io
+ All rights reserved. Contact: rednaga@protonmail.com
+
+
+ This file is part of APKiD
+
+
+ Commercial License Usage
+ ------------------------
+ Licensees holding valid commercial APKiD licenses may use this file
+ in accordance with the commercial license agreement provided with the
+ Software or, alternatively, in accordance with the terms contained in
+ a written agreement between you and RedNaga.
+
+
+ GNU General Public License Usage
+ --------------------------------
+ Alternatively, this file may be used under the terms of the GNU General
+ Public License version 3.0 as published by the Free Software Foundation
+ and appearing in the file LICENSE.GPL included in the packaging of this
+ file. Please visit http://www.gnu.org/copyleft/gpl.html and review the
+ information to ensure the GNU General Public License version 3.0
+ requirements will be met.
+"""
+
+import io
+import lzma
+import os
+import struct
+import sys
+import traceback
+import zipfile
+from typing import Union, IO, List, Dict, Set
+
+import yara
+
+from .output import OutputFormatter
+from .rules import RulesManager
+
+SCANNABLE_FILE_MAGICS: Dict[str, Set[bytes]] = {
+    'zip': {b'PK\x03\x04', b'PK\x05\x06', b'PK\x07\x08'},
+    'dex': {b'dex\n', b'dey\n'},
+    'elf': {b'\x7fELF'},
+    'res': {b'\x02\x00\x0c\x00'},
+    'dll': {b'MZ\x90\x00'},
+    # TODO: implement axml yara module
+    # 'axml': set(),
+}
+XZ_COMPRESSION_TYPE = 95
+ZIP_LFH_SIG_SIZE = 4
+ZIP_LFH_FIELDS_SIZE = 26
+ZIP_LFH_HEADER_SIZE = ZIP_LFH_SIG_SIZE + ZIP_LFH_FIELDS_SIZE
+
+
+class Options(object):
+
+    def __init__(self, timeout: int = 10, verbose: bool = False, json: bool = False, output_dir: Union[str, None] = None,
+                 typing: Union[str, None] = 'magic', entry_max_scan_size: int = 0, scan_depth=2, recursive: bool = False,
+                 include_types: bool = False):
+        """Scan options.
+        Holds user-supplied options governing how APKiD behaves.
+
+        Parameters
+        ----------
+        timeout : integer, optional (default=10)
+            The number of seconds before Yara match should time out.
+        json : boolean, optional (default=False)
+            If the output should be JSON format.
+        output_dir : string or None, optional (default=None)
+            Directory to write individual scan results to. If this is true, it implies `json_output=True`.
+            Note: This is useful for feature extraction of a bunch of APKs.
+        verbose : boolean, optional (default=False)
+            When set to `True`, log warnings and other debug information.
+        typing : string or None, optional (default="magic")
+            Determines how the scanner decides if a file should be scanned.
+            If "magic", then require the file match a built-in list of supported file magics
+            If "filename", then scan files which have names known to be supported (e.g. ".dex")
+            If None, pass every file to Yara for matching.
+            Note: This option defines a trade-off between performance and accuracy. For example, if you're scanning a large APK file, it is expensive
+            to uncompress every ZIP entry for Yara matching. It's much faster to only decompress files which have a known extension such as ".dex".
+            But in many cases files may not have the correct extension, e.g. a DEX file may be named "notmalware.gif". On the other extreme, one could
+            simply decompress every file and scan it, but this is wasteful because most files in an APK are typically not interesting, e.g. images.
+            The default behavior of "magic" only needs to read a few bytes to decide if a file should be completely uncompressed.
+        entry_max_scan_size : integer, optional (default=0)
+            If > 0, only scan APK entries if their uncompressed size is less than this value.
+        scan_depth : integer, optional (default=2)
+            Determines how many times scanner should recurse into nested archives.
+            If 0, don't recurse into nested archives.
+            Note: It's possible to construct a malicious ZIP which can be infinitely nested. It's therefore necessary to limit the scan depth.
+            Don't get cheeky and think you can set this value to 1000 and scan random malware without blowing up your memory.
+        recursive : boolean, optional (default=True)
+            If true, when scanning a directory, will recurse into subdirectories.
+        """
+        self.timeout = timeout
+        self.verbose = verbose
+        self.typing = typing
+        self.entry_max_scan_size = entry_max_scan_size
+        self.scan_depth = scan_depth
+        self.recursive = recursive
+        self.rules_manager = RulesManager()
+        self.output = OutputFormatter(
+            json_output=json,
+            output_dir=output_dir,
+            rules_manager=self.rules_manager,
+            include_types=include_types
+        )
+
+
+class Scanner(object):
+
+    def __init__(self, rules: yara.Rules, options: Options):
+        self.rules = rules
+        self.options = options
+
+    def scan(self, path: str) -> None:
+        if os.path.isfile(path):
+            results = self.scan_file(path)
+            if len(results) > 0:
+                self.options.output.write(results)
+        elif os.path.isdir(path):
+            self.scan_directory(path)
+
+    def scan_directory(self, dir_path: str) -> None:
+        for file_path in self._yield_file_paths(dir_path):
+            self.scan(file_path)
+
+    def scan_file(self, file_path: str) -> Dict[str, List[yara.Match]]:
+        results = []
+        with open(file_path, 'rb') as f:
+            try:
+                results: Dict[str, List[yara.Match]] = self.scan_file_obj(f, file_path)
+            except Exception as e:
+                stack = traceback.format_exc()
+                print(f"Exception scanning {file_path}: {stack}")
+        return results
+
+    def scan_file_obj(self, file: IO, file_path: str = '$FILE$'):
+        if file_path == '$FILE$':
+            file_name = file_path
+        else:
+            file_name = os.path.basename(file_path)
+
+        results: Dict[str, List[yara.Match]] = {}
+        if not self._should_scan(file, file_name):
+            return results
+
+        matches: List[yara.Matches] = self.rules.match(data=file.read(), timeout=self.options.timeout)
+        if len(matches) > 0:
+            results[file_path] = matches
+        if self._is_zipfile(file, file_name):
+            with zipfile.ZipFile(file) as zf:
+                zip_results = self._scan_zip(zf)
+            for entry_name, entry_matches in zip_results.items():
+                results[f'{file_path}!{entry_name}'] = entry_matches
+        return results
+
+    def _scan_zip(self, zf: zipfile.ZipFile, depth=0) -> Dict[str, List[yara.Match]]:
+        results: Dict[str, List[yara.Match]] = {}
+        for info in zf.infolist():
+            if info.is_dir():
+                continue
+            try:
+                self._scan_zip_entry(zf, info, results, depth)
+            except Exception as e:
+                stack = traceback.format_exc()
+                print(f"Exception scanning {info.filename} in {zf.filename}, depth={depth}: {stack}",
+                    file=sys.stderr)
+        return results
+
+    def _scan_zip_entry(self, zf, info, results, depth) -> None:
+        try:
+            with zf.open(info) as entry:
+                # Python 3.6 zip entries are not seek'able :(
+                entry_buffer: IO = io.BytesIO(entry.read(4))
+                entry_buffer.seek(0)
+                if not self._should_scan(entry_buffer, info.filename):
+                    return
+                entry_buffer.seek(4)
+                entry_buffer.write(entry.read())
+        except zipfile.BadZipFile as e:
+            if "Overlapped entries" in str(e) or "possible zip bomb" in str(e):
+                if self.options.verbose:
+                    print(f"[W] Skipping overlapped entry {info.filename} (possible zip bomb)")
+                return
+            else:
+                raise
+        except NotImplementedError:
+            # XZ-compression
+            if info.compress_type == XZ_COMPRESSION_TYPE:
+                with open(zf.filename, 'rb') as raw_zip:
+                    raw_zip.seek(info.header_offset + ZIP_LFH_FIELDS_SIZE)
+                    filename_len = struct.unpack('<H', raw_zip.read(2))[0]
+                    extra_len = struct.unpack('<H', raw_zip.read(2))[0]
+                    data_offset = info.header_offset + ZIP_LFH_HEADER_SIZE + filename_len + extra_len
+                    raw_zip.seek(data_offset)
+                    compressed_data = raw_zip.read(info.compress_size)
+                    try:
+                        decompressed_data = lzma.decompress(compressed_data)
+                        entry_buffer = io.BytesIO(decompressed_data)
+                    except Exception as e:
+                        print(f"[E] Failed to decompress {info.filename}: {e}")
+                        return
+            else:
+                print(f"[E] Unsupported compression method {info.compress_type} for {info.filename}")
+                return
+
+
+        entry_buffer.seek(0)
+        matches = self.rules.match(data=entry_buffer.read(), timeout=self.options.timeout)
+
+        if len(matches) > 0:
+            results[info.filename] = matches
+
+        if depth < self.options.scan_depth and self._is_zipfile(entry_buffer, info.filename):
+            with zipfile.ZipFile(entry_buffer) as zip_entry:
+                nested_results = self._scan_zip(zip_entry, depth=depth + 1)
+                for nested_name, nested_matches in nested_results.items():
+                    results[f'{info.filename}!{nested_name}'] = nested_matches
+
+    @staticmethod
+    def _type_file(file: IO) -> Union[None, str]:
+        magic = file.read(4)
+        file.seek(0)
+        for file_type, magics in SCANNABLE_FILE_MAGICS.items():
+            if magic in magics:
+                return file_type
+        return None
+
+    def _is_zipfile(self, file: IO, name: str) -> bool:
+        if self.options.typing == 'filename':
+            name = name.lower()
+            return name.endswith('.apk') or name.endswith('.zip') or name.endswith('.jar')
+        else:
+            # Look at file magic since zipfile.is_zipfile isn't perfect
+            # Some elfs may be considered zips and this causes apkid to barf errors
+            file.seek(0)
+            return Scanner._type_file(file) == 'zip' and zipfile.is_zipfile(file)
+
+    def _should_scan(self, file: IO, name: str) -> bool:
+        if self.options.typing == 'magic':
+            file_type = Scanner._type_file(file)
+            return file_type is not None
+        elif self.options.typing == 'filename':
+            name = name.lower()
+            return name.startswith('classes') \
+                   or name.startswith('AndroidManifest.xml') \
+                   or name.startswith('lib/') \
+                   or name.endswith('.so') \
+                   or name.endswith('.dex') \
+                   or name.endswith('.apk') \
+                   or name.endswith('.arsc') \
+                   or name.endswith('.dll')
+        return True
+
+    def _yield_file_paths(self, dir_path: str):
+        if self.options.recursive:
+            for root, _, filenames in os.walk(dir_path):
+                for path in filenames:
+                    yield os.path.join(root, path)
+        else:
+            for path in os.listdir(dir_path):
+                full_path = os.path.join(dir_path, path)
+                if os.path.isdir(full_path):
+                    continue
+                yield full_path

package/apkid/main.py

@@ -0,0 +1,98 @@
+"""
+ Copyright (C) 2023  RedNaga. https://rednaga.io
+ All rights reserved. Contact: rednaga@protonmail.com
+
+
+ This file is part of APKiD
+
+
+ Commercial License Usage
+ ------------------------
+ Licensees holding valid commercial APKiD licenses may use this file
+ in accordance with the commercial license agreement provided with the
+ Software or, alternatively, in accordance with the terms contained in
+ a written agreement between you and RedNaga.
+
+
+ GNU General Public License Usage
+ --------------------------------
+ Alternatively, this file may be used under the terms of the GNU General
+ Public License version 3.0 as published by the Free Software Foundation
+ and appearing in the file LICENSE.GPL included in the packaging of this
+ file. Please visit http://www.gnu.org/copyleft/gpl.html and review the
+ information to ensure the GNU General Public License version 3.0
+ requirements will be met.
+"""
+
+import argparse
+
+from apkid.apkid import Scanner, Options
+from . import __version__
+
+
+def get_parser():
+    formatter = lambda prog: argparse.ArgumentDefaultsHelpFormatter(prog, max_help_position=50, width=100)
+
+    parser = argparse.ArgumentParser(
+        description=f"APKiD - Android Application Identifier v{__version__}",
+        formatter_class=formatter
+    )
+    parser.add_argument('input', metavar='FILE', type=str, nargs='*',
+                        help="apk, dex, or directory")
+    parser.add_argument('-v', '--verbose', action='store_true',
+                        help="log debug messages")
+
+    scanning = parser.add_argument_group('scanning')
+    scanning.add_argument('-t', '--timeout', type=int, default=30,
+                          help="Yara scan timeout (in seconds)")
+    scanning.add_argument('-r', '--recursive', action='store_true', default=False,
+                          help="recurse into subdirectories")
+    scanning.add_argument('--scan-depth', type=int, default=2,
+                          help="how deep to go when scanning nested zips")
+    scanning.add_argument('--entry-max-scan-size', type=int, default=100 * 1024 * 1024,
+                          help="max zip entry size to scan in bytes, 0 = no limit")
+    scanning.add_argument('--typing', choices=('magic', 'filename', 'none'), default='magic',
+                          help="method to decide which files to scan")
+
+    output = parser.add_argument_group('output')
+    output.add_argument('-j', '--json', action='store_true',
+                        help="output scan results in JSON format", )
+    output.add_argument('-o', '--output-dir', metavar='DIR', default=None,
+                        help="write individual results here (implies --json)")
+    output.add_argument('--include-types', action='store_true', default=False,
+                        help="include file type info for matched files")
+
+    return parser
+
+
+def build_options(args) -> Options:
+    return Options(
+        timeout=args.timeout,
+        verbose=args.verbose,
+        json=args.json,
+        output_dir=args.output_dir,
+        typing=args.typing,
+        entry_max_scan_size=args.entry_max_scan_size,
+        scan_depth=args.scan_depth,
+        recursive=args.recursive,
+        include_types=args.include_types,
+    )
+
+
+def main():
+    parser = get_parser()
+    args = parser.parse_args()
+    options = build_options(args)
+
+    if not options.output.json:
+        print(f"[+] APKiD {__version__} :: from RedNaga :: rednaga.io")
+
+    rules = options.rules_manager.load()
+    scanner = Scanner(rules, options)
+
+    for input in args.input:
+        scanner.scan(input)
+
+
+if __name__ == '__main__':
+    main()

package/apkid/output.py

@@ -0,0 +1,177 @@
+"""
+ Copyright (C) 2023  RedNaga. https://rednaga.io
+ All rights reserved. Contact: rednaga@protonmail.com
+
+
+ This file is part of APKiD
+
+
+ Commercial License Usage
+ ------------------------
+ Licensees holding valid commercial APKiD licenses may use this file
+ in accordance with the commercial license agreement provided with the
+ Software or, alternatively, in accordance with the terms contained in
+ a written agreement between you and RedNaga.
+
+
+ GNU General Public License Usage
+ --------------------------------
+ Alternatively, this file may be used under the terms of the GNU General
+ Public License version 3.0 as published by the Free Software Foundation
+ and appearing in the file LICENSE.GPL included in the packaging of this
+ file. Please visit http://www.gnu.org/copyleft/gpl.html and review the
+ information to ensure the GNU General Public License version 3.0
+ requirements will be met.
+"""
+
+import json
+import os
+import sys
+from typing import Dict, List, Union
+
+import yara
+
+from .rules import RulesManager
+
+prt_red = lambda s: f"\033[91m{s}\033[00m"
+prt_green = lambda s: f"\033[92m{s}\033[00m"
+prt_yellow = lambda s: f"\033[93m{s}\033[00m"
+prt_light_purple = lambda s: f"\033[94m{s}\033[00m"
+prt_purple = lambda s: f"\033[95m{s}\033[00m"
+prt_cyan = lambda s: f"\033[36m{s}\033[00m"
+prt_light_cyan = lambda s: f"\033[96m{s}\033[00m"
+prt_light_gray = lambda s: f"\033[97m{s}\033[00m"
+prt_orange = lambda s: f"\033[33m{s}\033[00m"
+prt_pink = lambda s: f"\033[35m{s}\033[00m"
+
+def is_windows_cmd():
+    """
+    Check if the current environment is a Windows command prompt.
+
+    Returns:
+        bool: True if the operating system is Windows and the SESSIONNAME
+              environment variable is not set, indicating a command prompt.
+    """
+    return os.name == 'nt' and (
+        os.getenv('SESSIONNAME') is None
+    )
+
+def colorize_tag(tag) -> str:
+    if tag == 'compiler':
+        return prt_cyan(tag)
+    elif tag == 'manipulator':
+        return prt_light_cyan(tag)
+    elif tag == 'abnormal':
+        return prt_light_gray(tag)
+    elif tag in ['anti_vm', 'anti_disassembly', 'anti_debug', 'anti_root']:
+        return prt_purple(tag)
+    elif tag in ['packer', 'protector', 'anticheat']:
+        return prt_red(tag)
+    elif tag == 'obfuscator':
+        return prt_yellow(tag)
+    elif tag == 'dropper':
+        return prt_green(tag)
+    elif tag == 'embedded':
+        return prt_light_purple(tag)
+    elif tag == 'file_type':
+        return prt_orange(tag)
+    elif tag == 'internal':
+        return prt_pink(tag)
+    else:
+        return tag
+
+
+class OutputFormatter(object):
+    def __init__(self, json_output: bool, output_dir: Union[str, None], rules_manager: RulesManager, include_types: bool):
+        from apkid import __version__
+        self.output_dir = output_dir
+        self.json = json_output or output_dir
+        self.version = __version__
+        self.rules_hash = rules_manager.hash
+        self.include_types = include_types
+
+    def write(self, results: Dict[str, List[yara.Match]]) -> None:
+        """
+         Example yara.Match:
+        {
+          'tags': ['foo', 'bar'],
+          'matches': True,
+          'namespace': 'default',
+          'rule': 'my_rule',
+          'meta': {},
+          'strings': [(81L, '$a', 'abc'), (141L, '$b', 'def')]
+        }
+        """
+
+        if self.output_dir:
+            # Result keys are file paths. Shortest key is base file in the case of archives.
+            base_file = sorted(results.keys(), key=lambda k: len(k))[0]
+            out_file = os.path.join(self.output_dir, *base_file.split(os.path.sep))
+            out_path = os.path.dirname(out_file)
+            if not os.path.exists(out_path):
+                os.makedirs(out_path)
+            output = self.build_json_output(results)
+            with open(out_file, 'w') as f:
+                f.write(json.dumps(output))
+        else:
+            if self.json:
+                self._print_json(results)
+            else:
+                self._print_console(results)
+
+    def build_json_output(self, results: Dict[str, List[yara.Match]]):
+        output = {
+            'apkid_version': self.version,
+            'rules_sha256': self.rules_hash,
+            'files': [],
+        }
+        for filename, matches in results.items():
+            match_results = self._build_match_results(matches)
+            if len(match_results) == 0:
+                continue
+            result = {
+                'filename': filename,
+                'matches': match_results,
+            }
+            output['files'].append(result)
+        return output
+
+    def _print_json(self, results: Dict[str, List[yara.Match]]) -> None:
+        output = self.build_json_output(results)
+        print(json.dumps(output, sort_keys=True))
+
+    def _print_console(self, results: Dict[str, List[yara.Match]]) -> None:
+        for key, raw_matches in results.items():
+            match_results = self._build_match_results(raw_matches)
+            if len(match_results) == 0:
+                continue
+            print(f"[*] {key}")
+            for tags in sorted(match_results):
+                descriptions = ', '.join(sorted(match_results[tags]))
+                if sys.stdout.isatty() and not is_windows_cmd():
+                    tags_str = OutputFormatter._colorize_tags(tags)
+                else:
+                    tags_str = tags
+                print(f" |-> {tags_str} : {descriptions}")
+
+    def _build_match_results(self, matches) -> Dict[str, List[str]]:
+        results: Dict[str, List[str]] = {}
+        for m in matches:
+            if 'file_type' in m.tags and not self.include_types:
+                continue
+            tags = ', '.join(sorted(m.tags))
+            description = m.meta.get('description', m)
+            if tags in results:
+                if description not in results[tags]:
+                    results[tags].append(description)
+            else:
+                results[tags] = [description]
+        return results
+
+    @staticmethod
+    def _colorize_tags(tags) -> str:
+        colored_tags = []
+        for tag in tags.split(', '):
+            colored_tag = colorize_tag(tag)
+            colored_tags.append(colored_tag)
+        return ', '.join(colored_tags)

package/apkid/rules/apk/common.yara

@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2023  RedNaga. https://rednaga.io
+ * All rights reserved. Contact: rednaga@protonmail.com
+ *
+ *
+ * This file is part of APKiD
+ *
+ *
+ * Commercial License Usage
+ * ------------------------
+ * Licensees holding valid commercial APKiD licenses may use this file
+ * in accordance with the commercial license agreement provided with the
+ * Software or, alternatively, in accordance with the terms contained in
+ * a written agreement between you and RedNaga.
+ *
+ *
+ * GNU General Public License Usage
+ * --------------------------------
+ * Alternatively, this file may be used under the terms of the GNU General
+ * Public License version 3.0 as published by the Free Software Foundation
+ * and appearing in the file LICENSE.GPL included in the packaging of this
+ * file. Please visit http://www.gnu.org/copyleft/gpl.html and review the
+ * information to ensure the GNU General Public License version 3.0
+ * requirements will be met.
+ *
+ **/
+
+rule is_apk : file_type
+{
+  meta:
+    description = "APK"
+
+  strings:
+    $zip_head = "PK"
+    $manifest = "AndroidManifest.xml"
+
+  condition:
+    $zip_head at 0 and $manifest and #manifest >= 2
+}
+
+private rule is_signed_apk : internal
+{
+  meta:
+    description = "Resembles a signed APK that is likely not corrupt"
+
+  strings:
+    $meta_inf = "META-INF/"
+    $ext_rsa = ".RSA"
+    $ext_dsa = ".DSA"
+    $ext_ec = ".EC"
+    $apk_sig_block_footer = { 41 50 4B 20 53 69 67 20 42 6C 6F 63 6B 20 34 32 50 4B 01 02 }
+
+  condition:
+    is_apk and
+    (
+      for all of ($meta_inf*) : ($ext_rsa or $ext_dsa or $ext_ec in (@ + 9..@ + 9 + 100)) or
+      $apk_sig_block_footer
+    )
+}
+
+private rule is_unsigned_apk : internal
+{
+  meta:
+    description = "Resembles an unsigned APK that is likely not corrupt"
+
+  condition:
+    is_apk and not is_signed_apk
+}