rbbt-dm 1.2.1 → 1.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ab775c0224960820e5c62e294e6a183be49201da15710b66544762e1aaf97ebf
-  data.tar.gz: 8fffb47ba226f06d1f41a8893d085bdc12c33c021cf2f0152f4cc741db36e420
+  metadata.gz: 2ff72107967b0f7c654697f3a7b3c0ef10f7a5264d775117f12f74084a2819b2
+  data.tar.gz: 6b9a58b5a2723c095332f79a37d9c1c7f4bc1431410f23a55beeed1c3b52f7ad
 SHA512:
-  metadata.gz: 8be084156063cd93c7fe905bc4b6248dd376bbfcff8e650cbb03a4cc5c28f29dbcdaa2801895a3663067d7660e8bc2cf96682829519ebd6511a7a74cec021da0
-  data.tar.gz: 9c8570722319caf5afe60c90778d0b8517e70064030e0081968d919a314cfe1af90f63d13a7b8ce8dd56da6644db89916a729a4005f1e20745c8d4b45d50394c
+  metadata.gz: a0fb4198cb0be3aa5253df0f655ee230621dd26a31956a774fffe95eac35f4c8b558a41c0e340c25c6eef463760ff6230b967f09eb09671b2078a50066067384
+  data.tar.gz: 0bd6c3667a8ec26ed092c54e78176671807ed0634e136e497303e34a17b2740e5e023041bc06389ac187de39b54942b9b1c5cd77abbc067c89250424654b6974

data/lib/rbbt/vector/model.rb

@@ -436,7 +436,7 @@ cat(paste(label, sep="\\n", collapse="\\n"));
     ensure
       @features = orig_features
       @labels = orig_labels
-    end
+    end unless folds == -1
     self.train unless folds == 1
     res
   end

data/python/rbbt_dm/huggingface.py

@@ -0,0 +1,112 @@
+#{{{ LOAD MODEL
+
+def import_module_class(module, class_name):
+    exec(f"from {module} import {class_name}")
+    return eval(class_name)
+
+def load_model(task, checkpoint):
+    class_name = 'AutoModelFor' + task
+    return import_module_class('transformers', class_name).from_pretrained(checkpoint)
+
+def load_tokenizer(task, checkpoint):
+    class_name = 'AutoTokenizer'
+    return import_module_class('transformers', class_name).from_pretrained(checkpoint)
+
+def load_model_and_tokenizer(task, checkpoint):
+    model = load_model(task, checkpoint)
+    tokenizer = load_tokenizer(task, checkpoint)
+    return model, tokenizer
+
+#{{{ SIMPLE EVALUATE
+
+def forward(model, features):
+    return model(**features)
+
+def logits(predictions):
+    logits = predictions["logits"]
+    return [v.detach().cpu().numpy() for v in logits]
+
+def eval_model(model, tokenizer, texts, return_logits = True):
+    features = tokenizer(texts, return_tensors='pt', truncation=True).to(model.device)
+    predictions = forward(model, features)
+    if (return_logits):
+        return logits(predictions)
+    else:
+        return predictions
+
+#{{{ TRAIN AND PREDICT
+
+def load_tsv(tsv_file):
+    from datasets import load_dataset
+    return load_dataset('csv', data_files=[tsv_file], sep="\t")
+
+def tsv_dataset(tokenizer, tsv_file):
+    dataset = load_tsv(tsv_file)
+    tokenized_dataset = dataset.map(lambda example: tokenizer(example["text"], truncation=True) , batched=True)
+    return tokenized_dataset
+
+def training_args(*args, **kwargs):
+    from transformers import TrainingArguments
+    training_args = TrainingArguments(*args, **kwargs)
+    return training_args
+
+
+def train_model(model, tokenizer, training_args, tsv_file):
+    from transformers import Trainer
+
+    tokenized_dataset = tsv_dataset(tokenizer, tsv_file)
+
+    trainer = Trainer(
+            model,
+            training_args,
+            train_dataset = tokenized_dataset["train"],
+            tokenizer = tokenizer
+            )
+
+    trainer.train()
+
+def find_tokens_in_input(dataset, token_ids):
+    position_rows = []
+
+    for row in dataset:
+        input_ids = row["input_ids"]
+
+        if (not hasattr(token_ids, "__len__")):
+            token_ids = [token_ids]
+
+        positions = []
+        for token_id in token_ids:
+
+            item_positions = []
+            for i in range(len(input_ids)):
+                if input_ids[i] == token_id:
+                    item_positions.append(i)
+
+            positions.append(item_positions)
+
+
+        position_rows.append(positions)
+
+    return position_rows
+
+
+
+def predict_model(model, tokenizer, training_args, tsv_file, locate_tokens = None):
+    from transformers import Trainer
+
+    tokenized_dataset = tsv_dataset(tokenizer, tsv_file)
+
+    trainer = Trainer(
+            model,
+            training_args,
+            tokenizer = tokenizer
+            )
+
+    result = trainer.predict(test_dataset = tokenized_dataset["train"])
+    if (locate_tokens != None):
+        token_ids = tokenizer.convert_tokens_to_ids(locate_tokens)
+        token_positions = find_tokens_in_input(tokenized_dataset["train"], token_ids)
+        return dict(result=result, token_positions=token_positions)
+    else:
+        return result
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-dm
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.2.3
 platform: ruby
 authors:
 - Miguel Vazquez
@@ -113,6 +113,7 @@ files:
 - lib/rbbt/vector/model/svm.rb
 - lib/rbbt/vector/model/tensorflow.rb
 - lib/rbbt/vector/model/util.rb
+- python/rbbt_dm/huggingface.py
 - share/R/MA.R
 - share/R/barcode.R
 - share/R/heatmap.3.R