grf-file-extractor 1.0.35

package/README.md

@@ -0,0 +1,33 @@
+Grf-extractor
+=========
+
+This is a command line tool for extracting and handling .grf packages.
+
+GRF files are a set of packed and compressed files, normally found on Ragnarok's game files.
+
+Install
+=========
+```
+npm install -g grf-extractor
+```
+
+Usage
+====
+```
+Usage: grf-extractor -g data.grf -o output_dir
+Options:
+  -g, --grf          The grf file to be worked on.                                                                                        
+  -s, --search       Search a single file on the .grf or a list of files separated by comma. RegExp are supported.                        
+  -l, --list         List files inside the grf.                                                                                           
+  -o, --output       Output directoy to write the extracted files to.                                                                     
+  -e, --extract      Extract a single file from the .grf, prints to stdout. Example: grf-extractor -e data/clientinfo.xml > clientinfo.xml
+  -c, --concurrency  Concurrency rate, how many parallel extractions should we do, set it to higher values for a faster extraction.         [default: 100]
+  -v, --verbose      Enable verbose output, will print debug messages.                                                                    
+  -h, --help         Print help and usage information  
+```
+
+TODO
+======
+* Support writting files to the grf.
+* Standardize the API.
+* Refactor `extractor.js`.

package/dist/index.js

@@ -0,0 +1,32 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  sayHello: () => sayHello
+});
+module.exports = __toCommonJS(index_exports);
+function sayHello(name = "world") {
+  return `Hello, ${name}!`;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  sayHello
+});

package/dist/index.mjs

@@ -0,0 +1,7 @@
+// src/index.ts
+function sayHello(name = "world") {
+  return `Hello, ${name}!`;
+}
+export {
+  sayHello
+};

package/index.js

@@ -0,0 +1,192 @@
+#!/usr/bin/env node
+// Entry point
+// Parse arguments and setup the enviroment
+
+// Dependencies
+var argv = require('optimist')
+var _package = require('./package.json')
+var Grf  = require('./lib/grf.js')
+var ProgressBar = require('progress')
+var Fs = require('fs')
+
+// Setup args
+argv.options('g', {
+	alias: 'grf',
+	describe: 'The grf file to be worked on.',
+})
+argv.options('s', {
+	alias: 'search',
+	describe: 'Search a single file on the .grf or a list of files separated by comma. RegExp are supported.',
+})
+argv.options('l', {
+	alias: 'list',
+	describe: 'List files inside the grf.',
+})
+argv.options('o', {
+	alias: 'output',
+	describe: 'Output directoy to write the extracted files to.',
+})
+argv.options('e', {
+	alias: 'extract',
+	describe: 'Extract a single file from the .grf, prints to stdout. Example: grf-extractor -e data/clientinfo.xml > clientinfo.xml',
+})
+argv.options('c', {
+	alias: 'concurrency',
+	default: 100,
+	describe: 'Concurrency rate, how many parallel extractions should we do, set it to higher values for a faster extraction.',
+})
+argv.options('v', {
+	alias: 'verbose',
+	describe: 'Enable verbose output, will print debug messages.',
+})
+argv.options('version', {
+	describe: 'Print package version.',
+})
+argv.options('k', {
+	alias: 'key',
+	describe: 'Path to a cps.dll for decrypting encrypted GRF files.',
+})
+argv.options('h', {
+	alias: 'help',
+	describe: 'Print help and usage information',
+})
+
+
+// Parse arguments
+var arg = argv.argv
+
+// Print version
+if(arg.version) {
+	console.log(_package.version)
+	return
+}
+
+// Print help
+if(!arg.g || arg.h) {
+	console.error("Usage: grf-extractor -g data.grf -o output_dir [-k cps.dll]")
+	argv.showHelp()
+	return
+}
+
+// Init working grf
+var fd = Fs.openSync(arg.g, 'r')
+
+var grf = new Grf({
+	fd: fd
+})
+
+// Set debugging mode
+if(typeof arg.v !== 'undefined') {
+	grf.printDebug = true
+}
+
+// Search and print information about a single file in the grf 
+if(typeof arg.s === 'string') {
+	var list = arg.s.split(',')
+
+	for(var i in list) {
+		var pattern = list[i]
+
+		grf.searchPattern(new RegExp(pattern, 'gi'), function(matches) {
+			console.log("Search for %s:", pattern)
+			
+			if(! matches) {
+				console.log("- Nothing found matching this pattern.")
+				return
+			}
+
+			for(var m in matches) {
+				var match = matches[m]
+
+				console.log("- File path: %s - File size: %d kb", match.filename, match.real_size/1000)
+			}
+		})
+	}
+
+	return
+}
+
+// List files inside the grf
+if(typeof arg.l !== 'undefined') {
+	var list = grf.entries
+
+	for(var i in list) {
+		var file = list[i]
+
+		console.log("%s - size in bytes: %d", file.filename, file.real_size)
+	}
+
+	return
+}
+
+// Extract a single file frm the grf
+if(typeof arg.e === 'string') {
+	var filename = arg.e
+
+	// We need to replace our normal slashes for windows forward slash
+	filename = filename.replace( /\//g, '\\')
+
+	var exists = grf.getFile(filename, function(data){
+		var buf = Buffer.from(new Uint8Array(data))
+		process.stdout.write(buf)
+	})
+
+	if(exists === false) {
+		console.error("File '%s' not in the .grf, don't forget to put the data/ folder in front.", filename)
+		console.error("Example: -e data/clientinfo.xml")
+	}
+
+	return
+}
+
+// Load CPS decryption key if provided
+var Cluster = require('cluster')
+var cpsSbox = null
+if(typeof arg.k === 'string') {
+	var Cps = require('./lib/cps.js')
+	try {
+		cpsSbox = Cps.loadKey(arg.k)
+		if(Cluster.isMaster) {
+			console.log("Loaded CPS decryption key from %s", arg.k)
+		}
+	} catch(err) {
+		console.error("Failed to load CPS key:", err.message)
+		return
+	}
+}
+
+// Default action, extract the entire grf
+var extraction = grf.extract({
+	output: (typeof arg.o !== 'string' ? '' : arg.o),
+	concurrency: arg.c,
+	cpsSbox: cpsSbox,
+})
+
+extraction.on('start', function() {
+	console.log("Extraction has started on %d files.", grf.entries.length)
+	
+	// Pretty progress bar extraction on process
+	var bar = new ProgressBar('Progress [:current/:total] [:bar] :percent :elapseds - :etas', {
+		width: 100,
+		total: grf.entries.length
+	})
+	var last = 0
+	var timer = setInterval(function () {
+		var progress = extraction.progress()
+		var rate = progress - last
+		
+		last = progress
+		
+		bar.tick(rate)
+
+		if (bar.complete) {
+			console.log("Finished extracting GRF.")
+			process.nextTick(process.exit)
+		}
+	}, 100)
+})
+
+extraction.on('error', function(err) {
+	console.error("Extraction error", err)
+})
+

package/lib/Loaders/GameFile.js

@@ -0,0 +1,368 @@
+/**
+ * Loaders/GameFile.js
+ *
+ * Loaders for Gravity .grf file (Game RO File)
+ *
+ * This file is part of ROBrowser, Ragnarok Online in the Web Browser (http://www.robrowser.com/).
+ *
+ * @author Vincent Thibault
+ */
+
+'use strict';
+
+var GameFileDecrypt = require('./GameFileDecrypt');
+var BinaryReader = require('../Utils/BinaryReader');
+var Inflate = require('../Utils/Inflate');
+var Struct = require('../Utils/Struct');
+
+
+/**
+ * GRF Constructor
+ *
+ * @param {File} data
+ */
+function GRF( data )
+{
+	if (data) {
+		this.load( data );
+	}
+}
+
+
+/**
+ * @var {File System} Nodejs
+ */
+var fs = require('fs');
+
+
+/**
+ * GRF Constants
+ */
+GRF.FILELIST_TYPE_FILE           = 0x01; // entry is a file
+GRF.FILELIST_TYPE_ENCRYPT_MIXED  = 0x02; // encryption mode 0 (header DES + periodic DES/shuffle)
+GRF.FILELIST_TYPE_ENCRYPT_HEADER = 0x04; // encryption mode 1 (header DES only)
+
+
+/**
+ * GRF Structures
+ */
+GRF.struct_header = new Struct(
+	'unsigned char signature[15]',
+	'unsigned char key[15]',
+	'unsigned long file_table_offset',
+	'unsigned long skip',
+	'unsigned long filecount',
+	'unsigned long version'
+);
+
+GRF.struct_table = new Struct(
+	'unsigned long pack_size',
+	'unsigned long real_size'
+);
+
+GRF.struct_entry = new Struct(
+	'unsigned long pack_size',
+	'unsigned long length_aligned',
+	'unsigned long real_size',
+	'unsigned char type',
+	'unsigned long offset'
+);
+
+
+/**
+ * GRF METHODs
+ */
+GRF.prototype.file   = null;
+GRF.prototype.reader = null;
+GRF.prototype.header = null;
+GRF.prototype.table  = null;
+
+
+/**
+ * Loading GRF
+ *
+ * @param {File} file
+ */
+GRF.prototype.load   = function Load( file )
+{
+	// Global object
+	this.file   = file;
+
+	// node.js
+	this.reader = (typeof FileReaderSync !== 'undefined' ? new FileReaderSync() : {});
+
+
+	// Local object
+	var buffer, fp;
+	var header, entries, table;
+	var reader = this.reader;
+	var data, out;
+	var i, count;
+
+
+	// Load grf
+	file.slice  = file.slice || file.webkitSlice || file.mozSlice;
+	reader.load = function( start, len ) {
+		// node.js
+		if (fs && file.fd) {
+			var buffer = Buffer.alloc(len);
+			fs.readSync(file.fd, buffer, 0, len, start);
+			return buffer.buffer;
+		}
+
+		return reader.readAsArrayBuffer(
+			file.slice( start, start+len )
+		);
+	};
+
+
+	// Check if file has enought content.
+	if (file.size < GRF.struct_header.size) {
+		throw new Error('GRF::load() - Not enough bytes to be a valid GRF');
+	}
+
+	// Read the header
+	buffer = reader.load( 0, GRF.struct_header.size );
+	fp     = new BinaryReader(buffer);
+	header = fp.readStruct( GRF.struct_header );
+
+	header.signature  = String.fromCharCode.apply( null, header.signature);
+	header.filecount -= header.skip + 7;
+
+
+	// Check file header
+	if (header.signature !== 'Master of Magic') {
+		throw new Error('GRF::load() - Incorrect header "' + header.signature + '", must be "Master of Magic".');
+	}
+
+	if (header.version !== 0x200) {
+		throw new Error('GRF::load() - Incorrect version "0x' + parseInt(header.version, 10).toString(16) + '", just support version "0x200"');
+	}
+
+	if (header.file_table_offset + GRF.struct_header.size > file.size || header.file_table_offset < 0) {
+		throw new Error('GRF::load() - Can\'t jump to table list (' + header.file_table_offset + '), file length: ' + file.size);
+	}
+
+	// Load Table Info
+	buffer = reader.load( header.file_table_offset + GRF.struct_header.size, GRF.struct_table.size );
+	fp     = new BinaryReader( buffer );
+	table  = fp.readStruct( GRF.struct_table );
+
+	// Load Table Data
+	buffer = reader.load( header.file_table_offset + GRF.struct_header.size + GRF.struct_table.size, table.pack_size );
+	data   = new Uint8Array(buffer);
+	out    = new Uint8Array(table.real_size);
+
+	// Uncompress data
+	(new Inflate(data)).getBytes(out);
+
+	// Load entries
+	entries = loadEntries(out, header.filecount);
+
+	// Store table data (used for regex search in tablelist)
+	// Set filename to lowercase (case insensitive in official client)
+	table.data = '';
+	for (i = 0, count = entries.length; i < count; ++i) {
+		table.data          += entries[i].filename + '\0';
+		entries[i].filename  = entries[i].filename.toLowerCase();
+	}
+
+	// Sort entries (for binary search)
+	entries.sort(sortEntries);
+
+	this.header  = header;
+	this.entries = entries;
+	this.table   = table;
+};
+
+
+/**
+ * Load entries
+ * Note: this function is quiet intensive, BinaryReader is slowing down
+ * the process and generate too much memory to garbage (GC pause of 6sec).
+ *
+ * @param {Uint8Array} content table
+ * @param {number} file count
+ */
+ function loadEntries( out, count )
+ {
+	// Read all entries
+	var i, pos, str;
+	var entries = new Array(count);
+
+	for (i = 0, pos = 0; i < count; ++i) {
+		str = '';
+		while (out[pos]) {
+			str += String.fromCharCode(out[pos++]);
+		}
+		pos++;
+
+		entries[i] = {
+			filename:       str,
+			pack_size:      (out[pos++] | out[pos++] << 8 | out[pos++] << 16 | out[pos++] << 24) >>> 0,
+			length_aligned: (out[pos++] | out[pos++] << 8 | out[pos++] << 16 | out[pos++] << 24) >>> 0,
+			real_size:      (out[pos++] | out[pos++] << 8 | out[pos++] << 16 | out[pos++] << 24) >>> 0,
+			type:           out[pos++],
+			offset:         (out[pos++] | out[pos++] << 8 | out[pos++] << 16 | out[pos++] << 24) >>> 0
+		};
+	}
+
+	return entries;
+}
+
+
+
+/**
+ * Sort entries (to find it faster)
+ *
+ * @param {object} entry 1
+ * @param {object} entry 2
+ */
+function sortEntries(a, b)
+{
+	if (a.filename > b.filename) {
+		return 1;
+	}
+
+	if (a.filename < b.filename) {
+		return -1;
+	}
+
+	return 0;
+}
+
+
+/**
+ * Decode entry to return its content
+ *
+ * @param {ArrayBuffer}
+ * @param {Entry}
+ * @param {function} callback
+ */
+GRF.prototype.decodeEntry = function DecodeEntry( buffer, entry, callback )
+{
+	var out;
+
+	var data = new Uint8Array( buffer );
+
+	// Decode the file
+	if (entry.type & GRF.FILELIST_TYPE_ENCRYPT_MIXED) {
+		console.log(11111111)
+		GameFileDecrypt.decodeFull( data, entry.length_aligned, entry.pack_size);
+	}
+	else if (entry.type & GRF.FILELIST_TYPE_ENCRYPT_HEADER) {
+		console.log(22222222)
+		GameFileDecrypt.decodeHeader( data, entry.length_aligned );
+	}
+
+	// Uncompress
+	out = new Uint8Array(entry.real_size);
+	(new Inflate(data)).getBytes(out);
+
+	callback(out.buffer);
+};
+
+
+/**
+ * Binary Search hack to find files in GRF list
+ *
+ * @param {string} filename
+ */
+GRF.prototype.search = function searchClosure()
+{
+	var range = new Uint32Array(2);
+
+	return function search( filename )
+	{
+		var entries = this.entries;
+		var v       = 0;
+		var middle  = 0;
+
+		range[1] = 0;
+		range[0] = entries.length - 1;
+
+		while (range[1] < range[0]) {
+			middle   = range[1] + ((range[0]-range[1]) >> 1);
+			v        = (entries[middle].filename < filename) | 0;
+			range[v] = middle + v;
+		}
+
+		if (range[1] < entries.length && entries[range[1]].filename === filename) {
+			return range[1];
+		}
+
+		return -1;
+	};
+}();
+
+
+
+/**
+ * Find a file in the GRF
+ *
+ * @param {string} filename
+ * @param {function} callback
+ */
+GRF.prototype.getFile = function getFile( filename, callback )
+{
+	// Not case sensitive...
+	var path = filename.toLowerCase();
+	var entry, blob;
+	var reader;
+
+	var pos = this.search(path);
+
+	// If filename is find in GRF table list
+	if (pos !== -1) {
+		entry = this.entries[pos];
+
+		// Directory ?
+		if (!(entry.type & GRF.FILELIST_TYPE_FILE)) {
+			return false;
+		}
+
+		// node.js
+		if (fs && this.file.fd) {
+			var buffer = new Buffer(entry.length_aligned);
+			var grf = this;
+
+			fs.read(this.file.fd, buffer, 0, entry.length_aligned, entry.offset + GRF.struct_header.size, function(err, bytesRead, buffer){
+				if(err) throw err;
+				grf.decodeEntry( (new Uint8Array(buffer)).buffer, entry, callback);
+			});
+			return true;
+		}
+
+		blob = this.file.slice(
+			entry.offset + GRF.struct_header.size,
+			entry.offset + GRF.struct_header.size + entry.length_aligned
+		);
+
+		// Load into memory
+		if (self.FileReader) {
+			var grf = this;
+
+			reader = new FileReader();
+			reader.onload = function(){
+				grf.decodeEntry( reader.result, entry, callback);
+			};
+			reader.readAsArrayBuffer(blob);
+		}
+
+		// Firefox doesn't seems to support FileReader in web worker
+		else {
+			reader = new FileReaderSync();
+			this.decodeEntry( reader.readAsArrayBuffer(blob), entry, callback );
+		}
+
+		return true;
+	}
+
+	return false;
+};
+
+
+/**
+ * Export
+ */
+module.exports = GRF;